{ "random": { "model_size": "NA", "model_link": "https://seaeval.github.io/", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.242436974789916, "language_acc": { "Spanish": 0.22941176470588234, "English": 0.23781512605042016, "Chinese": 0.2605042016806723, "Vietnamese": 0.24201680672268908 }, "consistency_score_2": 0.2533613445378151, "consistency_score_3": 0.06932773109243698, "consistency_score_4": 0.024369747899159664, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.23781512605042016, "Spanish,Chinese": 0.2529411764705882, "Spanish,Vietnamese": 0.25798319327731095, "English,Chinese": 0.26638655462184874, "English,Vietnamese": 0.2504201680672269, "Chinese,Vietnamese": 0.2546218487394958 }, "3_combine": { "Spanish,English,Chinese": 0.07226890756302522, "Spanish,English,Vietnamese": 0.06050420168067227, "Spanish,Chinese,Vietnamese": 0.07058823529411765, "English,Chinese,Vietnamese": 0.07394957983193277 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.024369747899159664 } }, "AC3_2": 0.24777880638785366, "AC3_3": 0.10782237419812744, "AC3_4": 0.044287699315102014 }, "prompt_2": { "overall_acc": 0.2640756302521008, "language_acc": { "Spanish": 0.2571428571428571, "English": 0.25546218487394956, "Chinese": 0.26974789915966385, "Vietnamese": 0.2739495798319328 }, "consistency_score_2": 0.24439775910364148, "consistency_score_3": 0.05882352941176471, "consistency_score_4": 0.013445378151260505, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.22941176470588234, "Spanish,Chinese": 0.246218487394958, "Spanish,Vietnamese": 0.246218487394958, "English,Chinese": 0.24789915966386555, "English,Vietnamese": 0.25126050420168067, "Chinese,Vietnamese": 0.2453781512605042 }, "3_combine": { "Spanish,English,Chinese": 0.05966386554621849, "Spanish,English,Vietnamese": 0.05714285714285714, "Spanish,Chinese,Vietnamese": 0.06638655462184874, "English,Chinese,Vietnamese": 0.052100840336134456 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.013445378151260505 } }, "AC3_2": 0.2538559287697504, "AC3_3": 0.09621493356888625, "AC3_4": 0.02558794902353448 }, "prompt_3": { "overall_acc": 0.24726890756302522, "language_acc": { "Spanish": 0.2495798319327731, "English": 0.24033613445378152, "Chinese": 0.25798319327731095, "Vietnamese": 0.2411764705882353 }, "consistency_score_2": 0.24663865546218489, "consistency_score_3": 0.0592436974789916, "consistency_score_4": 0.01764705882352941, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.24369747899159663, "Spanish,Chinese": 0.24789915966386555, "Spanish,Vietnamese": 0.24873949579831933, "English,Chinese": 0.24705882352941178, "English,Vietnamese": 0.24705882352941178, "Chinese,Vietnamese": 0.2453781512605042 }, "3_combine": { "Spanish,English,Chinese": 0.06134453781512605, "Spanish,English,Vietnamese": 0.0546218487394958, "Spanish,Chinese,Vietnamese": 0.06050420168067227, "English,Chinese,Vietnamese": 0.06050420168067227 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.01764705882352941 } }, "AC3_2": 0.24695337934514394, "AC3_3": 0.09558578740236284, "AC3_4": 0.03294304239088608 }, "prompt_4": { "overall_acc": 0.25168067226890756, "language_acc": { "Spanish": 0.22941176470588234, "English": 0.26974789915966385, "Chinese": 0.27310924369747897, "Vietnamese": 0.23445378151260504 }, "consistency_score_2": 0.24985994397759104, "consistency_score_3": 0.06491596638655461, "consistency_score_4": 0.015126050420168067, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.26554621848739496, "Spanish,Chinese": 0.21260504201680672, "Spanish,Vietnamese": 0.24033613445378152, "English,Chinese": 0.24789915966386555, "English,Vietnamese": 0.2722689075630252, "Chinese,Vietnamese": 0.2605042016806723 }, "3_combine": { "Spanish,English,Chinese": 0.058823529411764705, "Spanish,English,Vietnamese": 0.073109243697479, "Spanish,Chinese,Vietnamese": 0.05630252100840336, "English,Chinese,Vietnamese": 0.07142857142857142 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.015126050420168067 } }, "AC3_2": 0.25076700320480716, "AC3_3": 0.10321078660454978, "AC3_4": 0.028537021096959894 }, "prompt_5": { "overall_acc": 0.2548319327731092, "language_acc": { "Spanish": 0.2453781512605042, "English": 0.2546218487394958, "Chinese": 0.27563025210084036, "Vietnamese": 0.24369747899159663 }, "consistency_score_2": 0.24915966386554622, "consistency_score_3": 0.06365546218487395, "consistency_score_4": 0.01680672268907563, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.2630252100840336, "Spanish,Chinese": 0.246218487394958, "Spanish,Vietnamese": 0.23529411764705882, "English,Chinese": 0.253781512605042, "English,Vietnamese": 0.2495798319327731, "Chinese,Vietnamese": 0.24705882352941178 }, "3_combine": { "Spanish,English,Chinese": 0.06722689075630252, "Spanish,English,Vietnamese": 0.05798319327731093, "Spanish,Chinese,Vietnamese": 0.06638655462184874, "English,Chinese,Vietnamese": 0.06302521008403361 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.01680672268907563 } }, "AC3_2": 0.2519638784568071, "AC3_3": 0.10186553510027453, "AC3_4": 0.03153372716835841 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.26285714285714284, "language_acc": { "Filipino": 0.32666666666666666, "Vietnamese": 0.29333333333333333, "Chinese": 0.23333333333333334, "Spanish": 0.26, "Malay": 0.24, "Indonesian": 0.25333333333333335, "English": 0.23333333333333334 }, "consistency_score_2": 0.24730158730158736, "consistency_score_3": 0.058666666666666666, "consistency_score_4": 0.013142857142857138, "consistency_score_5": 0.0031746031746031746, "consistency_score_6": 0.0009523809523809525, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.2, "Filipino,Chinese": 0.25333333333333335, "Filipino,Spanish": 0.2733333333333333, "Filipino,Malay": 0.24, "Filipino,Indonesian": 0.23333333333333334, "Filipino,English": 0.2733333333333333, "Vietnamese,Chinese": 0.29333333333333333, "Vietnamese,Spanish": 0.2733333333333333, "Vietnamese,Malay": 0.26, "Vietnamese,Indonesian": 0.25333333333333335, "Vietnamese,English": 0.21333333333333335, "Chinese,Spanish": 0.25333333333333335, "Chinese,Malay": 0.28, "Chinese,Indonesian": 0.23333333333333334, "Chinese,English": 0.26, "Spanish,Malay": 0.19333333333333333, "Spanish,Indonesian": 0.24666666666666667, "Spanish,English": 0.24666666666666667, "Malay,Indonesian": 0.2733333333333333, "Malay,English": 0.24, "Indonesian,English": 0.2 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.08, "Filipino,Vietnamese,Spanish": 0.06, "Filipino,Vietnamese,Malay": 0.06, "Filipino,Vietnamese,Indonesian": 0.06, "Filipino,Vietnamese,English": 0.04666666666666667, "Filipino,Chinese,Spanish": 0.06666666666666667, "Filipino,Chinese,Malay": 0.08, "Filipino,Chinese,Indonesian": 0.05333333333333334, "Filipino,Chinese,English": 0.07333333333333333, "Filipino,Spanish,Malay": 0.04, "Filipino,Spanish,Indonesian": 0.04, "Filipino,Spanish,English": 0.09333333333333334, "Filipino,Malay,Indonesian": 0.05333333333333334, "Filipino,Malay,English": 0.05333333333333334, "Filipino,Indonesian,English": 0.03333333333333333, "Vietnamese,Chinese,Spanish": 0.09333333333333334, "Vietnamese,Chinese,Malay": 0.08666666666666667, "Vietnamese,Chinese,Indonesian": 0.05333333333333334, "Vietnamese,Chinese,English": 0.07333333333333333, "Vietnamese,Spanish,Malay": 0.06, "Vietnamese,Spanish,Indonesian": 0.05333333333333334, "Vietnamese,Spanish,English": 0.04, "Vietnamese,Malay,Indonesian": 0.07333333333333333, "Vietnamese,Malay,English": 0.04666666666666667, "Vietnamese,Indonesian,English": 0.04, "Chinese,Spanish,Malay": 0.06666666666666667, "Chinese,Spanish,Indonesian": 0.04666666666666667, "Chinese,Spanish,English": 0.04, "Chinese,Malay,Indonesian": 0.07333333333333333, "Chinese,Malay,English": 0.06666666666666667, "Chinese,Indonesian,English": 0.04, "Spanish,Malay,Indonesian": 0.04666666666666667, "Spanish,Malay,English": 0.06666666666666667, "Spanish,Indonesian,English": 0.05333333333333334, "Malay,Indonesian,English": 0.04 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.02666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.02, "Filipino,Vietnamese,Chinese,Indonesian": 0.02666666666666667, "Filipino,Vietnamese,Chinese,English": 0.02666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.02, "Filipino,Vietnamese,Spanish,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Spanish,English": 0.006666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.02, "Filipino,Vietnamese,Malay,English": 0.013333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.013333333333333334, "Filipino,Chinese,Spanish,English": 0.013333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.013333333333333334, "Filipino,Chinese,Malay,English": 0.013333333333333334, "Filipino,Chinese,Indonesian,English": 0.006666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.0, "Filipino,Spanish,Malay,English": 0.02, "Filipino,Spanish,Indonesian,English": 0.02, "Filipino,Malay,Indonesian,English": 0.013333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.02666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, "Vietnamese,Chinese,Spanish,English": 0.006666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, "Vietnamese,Chinese,Malay,English": 0.02, "Vietnamese,Chinese,Indonesian,English": 0.0, "Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, "Vietnamese,Spanish,Malay,English": 0.013333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.013333333333333334, "Vietnamese,Malay,Indonesian,English": 0.013333333333333334, "Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, "Chinese,Spanish,Malay,English": 0.02, "Chinese,Spanish,Indonesian,English": 0.006666666666666667, "Chinese,Malay,Indonesian,English": 0.006666666666666667, "Spanish,Malay,Indonesian,English": 0.006666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Spanish,Malay,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Malay,Indonesian,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Chinese,Spanish,Malay,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.006666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 } }, "AC3_2": 0.2548422081463306, "AC3_3": 0.09592417058627951, "AC3_4": 0.025034013596371876, "AC3_5": 0.006273440161296806, "AC3_6": 0.0018978855072743965, "AC3_7": 0.0 }, "prompt_2": { "overall_acc": 0.2580952380952381, "language_acc": { "Filipino": 0.2733333333333333, "Vietnamese": 0.23333333333333334, "Chinese": 0.26, "Spanish": 0.22666666666666666, "Malay": 0.25333333333333335, "Indonesian": 0.29333333333333333, "English": 0.26666666666666666 }, "consistency_score_2": 0.24000000000000007, "consistency_score_3": 0.056761904761904756, "consistency_score_4": 0.013142857142857138, "consistency_score_5": 0.0022222222222222222, "consistency_score_6": 0.0, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.2866666666666667, "Filipino,Chinese": 0.21333333333333335, "Filipino,Spanish": 0.24666666666666667, "Filipino,Malay": 0.23333333333333334, "Filipino,Indonesian": 0.25333333333333335, "Filipino,English": 0.26, "Vietnamese,Chinese": 0.23333333333333334, "Vietnamese,Spanish": 0.2, "Vietnamese,Malay": 0.2, "Vietnamese,Indonesian": 0.22, "Vietnamese,English": 0.17333333333333334, "Chinese,Spanish": 0.24, "Chinese,Malay": 0.28, "Chinese,Indonesian": 0.24, "Chinese,English": 0.26666666666666666, "Spanish,Malay": 0.28, "Spanish,Indonesian": 0.24666666666666667, "Spanish,English": 0.2733333333333333, "Malay,Indonesian": 0.23333333333333334, "Malay,English": 0.24, "Indonesian,English": 0.22 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.06, "Filipino,Vietnamese,Spanish": 0.07333333333333333, "Filipino,Vietnamese,Malay": 0.06, "Filipino,Vietnamese,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,English": 0.04666666666666667, "Filipino,Chinese,Spanish": 0.04666666666666667, "Filipino,Chinese,Malay": 0.06666666666666667, "Filipino,Chinese,Indonesian": 0.04666666666666667, "Filipino,Chinese,English": 0.06666666666666667, "Filipino,Spanish,Malay": 0.06666666666666667, "Filipino,Spanish,Indonesian": 0.06, "Filipino,Spanish,English": 0.08, "Filipino,Malay,Indonesian": 0.05333333333333334, "Filipino,Malay,English": 0.06666666666666667, "Filipino,Indonesian,English": 0.03333333333333333, "Vietnamese,Chinese,Spanish": 0.04, "Vietnamese,Chinese,Malay": 0.05333333333333334, "Vietnamese,Chinese,Indonesian": 0.04666666666666667, "Vietnamese,Chinese,English": 0.05333333333333334, "Vietnamese,Spanish,Malay": 0.05333333333333334, "Vietnamese,Spanish,Indonesian": 0.07333333333333333, "Vietnamese,Spanish,English": 0.02666666666666667, "Vietnamese,Malay,Indonesian": 0.03333333333333333, "Vietnamese,Malay,English": 0.02666666666666667, "Vietnamese,Indonesian,English": 0.013333333333333334, "Chinese,Spanish,Malay": 0.07333333333333333, "Chinese,Spanish,Indonesian": 0.06, "Chinese,Spanish,English": 0.08666666666666667, "Chinese,Malay,Indonesian": 0.04666666666666667, "Chinese,Malay,English": 0.07333333333333333, "Chinese,Indonesian,English": 0.06666666666666667, "Spanish,Malay,Indonesian": 0.06666666666666667, "Spanish,Malay,English": 0.06, "Spanish,Indonesian,English": 0.06666666666666667, "Malay,Indonesian,English": 0.06666666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.013333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.02, "Filipino,Vietnamese,Chinese,Indonesian": 0.02, "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.02, "Filipino,Vietnamese,Spanish,Indonesian": 0.03333333333333333, "Filipino,Vietnamese,Spanish,English": 0.006666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Malay,English": 0.0, "Filipino,Vietnamese,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay": 0.006666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.013333333333333334, "Filipino,Chinese,Spanish,English": 0.013333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.006666666666666667, "Filipino,Chinese,Malay,English": 0.006666666666666667, "Filipino,Chinese,Indonesian,English": 0.013333333333333334, "Filipino,Spanish,Malay,Indonesian": 0.02, "Filipino,Spanish,Malay,English": 0.03333333333333333, "Filipino,Spanish,Indonesian,English": 0.006666666666666667, "Filipino,Malay,Indonesian,English": 0.013333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.013333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.02, "Vietnamese,Chinese,Spanish,English": 0.006666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, "Vietnamese,Chinese,Malay,English": 0.013333333333333334, "Vietnamese,Chinese,Indonesian,English": 0.0, "Vietnamese,Spanish,Malay,Indonesian": 0.013333333333333334, "Vietnamese,Spanish,Malay,English": 0.0, "Vietnamese,Spanish,Indonesian,English": 0.006666666666666667, "Vietnamese,Malay,Indonesian,English": 0.006666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.02, "Chinese,Spanish,Malay,English": 0.02666666666666667, "Chinese,Spanish,Indonesian,English": 0.02666666666666667, "Chinese,Malay,Indonesian,English": 0.02, "Spanish,Malay,Indonesian,English": 0.013333333333333334 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Chinese,Spanish,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Chinese,Spanish,Malay,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Spanish,Malay,Indonesian,English": 0.006666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Chinese,Spanish,Malay,Indonesian,English": 0.006666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 } }, "AC3_2": 0.24871892920436814, "AC3_3": 0.09305793215723321, "AC3_4": 0.02501203851405304, "AC3_5": 0.004406504063347907, "AC3_6": 0.0, "AC3_7": 0.0 }, "prompt_3": { "overall_acc": 0.24095238095238095, "language_acc": { "Filipino": 0.22666666666666666, "Vietnamese": 0.18, "Chinese": 0.28, "Spanish": 0.24666666666666667, "Malay": 0.21333333333333335, "Indonesian": 0.2733333333333333, "English": 0.26666666666666666 }, "consistency_score_2": 0.25968253968253974, "consistency_score_3": 0.06914285714285713, "consistency_score_4": 0.01923809523809524, "consistency_score_5": 0.005079365079365079, "consistency_score_6": 0.0009523809523809525, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.24666666666666667, "Filipino,Chinese": 0.26666666666666666, "Filipino,Spanish": 0.30666666666666664, "Filipino,Malay": 0.26, "Filipino,Indonesian": 0.20666666666666667, "Filipino,English": 0.26, "Vietnamese,Chinese": 0.25333333333333335, "Vietnamese,Spanish": 0.22666666666666666, "Vietnamese,Malay": 0.22666666666666666, "Vietnamese,Indonesian": 0.2866666666666667, "Vietnamese,English": 0.2733333333333333, "Chinese,Spanish": 0.22666666666666666, "Chinese,Malay": 0.22, "Chinese,Indonesian": 0.23333333333333334, "Chinese,English": 0.26, "Spanish,Malay": 0.2733333333333333, "Spanish,Indonesian": 0.24, "Spanish,English": 0.3, "Malay,Indonesian": 0.32666666666666666, "Malay,English": 0.24, "Indonesian,English": 0.32 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.08666666666666667, "Filipino,Vietnamese,Spanish": 0.07333333333333333, "Filipino,Vietnamese,Malay": 0.06666666666666667, "Filipino,Vietnamese,Indonesian": 0.04666666666666667, "Filipino,Vietnamese,English": 0.08, "Filipino,Chinese,Spanish": 0.06, "Filipino,Chinese,Malay": 0.07333333333333333, "Filipino,Chinese,Indonesian": 0.04666666666666667, "Filipino,Chinese,English": 0.07333333333333333, "Filipino,Spanish,Malay": 0.06666666666666667, "Filipino,Spanish,Indonesian": 0.05333333333333334, "Filipino,Spanish,English": 0.1, "Filipino,Malay,Indonesian": 0.07333333333333333, "Filipino,Malay,English": 0.06666666666666667, "Filipino,Indonesian,English": 0.08, "Vietnamese,Chinese,Spanish": 0.05333333333333334, "Vietnamese,Chinese,Malay": 0.05333333333333334, "Vietnamese,Chinese,Indonesian": 0.05333333333333334, "Vietnamese,Chinese,English": 0.08666666666666667, "Vietnamese,Spanish,Malay": 0.05333333333333334, "Vietnamese,Spanish,Indonesian": 0.06666666666666667, "Vietnamese,Spanish,English": 0.06, "Vietnamese,Malay,Indonesian": 0.06666666666666667, "Vietnamese,Malay,English": 0.04666666666666667, "Vietnamese,Indonesian,English": 0.09333333333333334, "Chinese,Spanish,Malay": 0.04666666666666667, "Chinese,Spanish,Indonesian": 0.07333333333333333, "Chinese,Spanish,English": 0.08, "Chinese,Malay,Indonesian": 0.05333333333333334, "Chinese,Malay,English": 0.06, "Chinese,Indonesian,English": 0.08, "Spanish,Malay,Indonesian": 0.09333333333333334, "Spanish,Malay,English": 0.06, "Spanish,Indonesian,English": 0.09333333333333334, "Malay,Indonesian,English": 0.1 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.02666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.02, "Filipino,Vietnamese,Chinese,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Chinese,English": 0.04, "Filipino,Vietnamese,Spanish,Malay": 0.013333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.02, "Filipino,Vietnamese,Spanish,English": 0.02666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Malay,English": 0.013333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.02, "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.02, "Filipino,Chinese,Spanish,English": 0.02, "Filipino,Chinese,Malay,Indonesian": 0.013333333333333334, "Filipino,Chinese,Malay,English": 0.02, "Filipino,Chinese,Indonesian,English": 0.02666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.02, "Filipino,Spanish,Malay,English": 0.013333333333333334, "Filipino,Spanish,Indonesian,English": 0.02666666666666667, "Filipino,Malay,Indonesian,English": 0.02666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.013333333333333334, "Vietnamese,Chinese,Spanish,English": 0.02666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.013333333333333334, "Vietnamese,Chinese,Malay,English": 0.02, "Vietnamese,Chinese,Indonesian,English": 0.03333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.02, "Vietnamese,Spanish,Malay,English": 0.006666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.02666666666666667, "Vietnamese,Malay,Indonesian,English": 0.006666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, "Chinese,Spanish,Malay,English": 0.006666666666666667, "Chinese,Spanish,Indonesian,English": 0.04666666666666667, "Chinese,Malay,Indonesian,English": 0.013333333333333334, "Spanish,Malay,Indonesian,English": 0.02 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.013333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Malay,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.013333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.006666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Chinese,Spanish,Malay,English": 0.0, "Filipino,Chinese,Spanish,Indonesian,English": 0.013333333333333334, "Filipino,Chinese,Malay,Indonesian,English": 0.006666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.013333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.013333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.006666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 } }, "AC3_2": 0.24996708634074552, "AC3_3": 0.10745173741708622, "AC3_4": 0.0356313183994689, "AC3_5": 0.009949001532054521, "AC3_6": 0.0018972628413604306, "AC3_7": 0.0 }, "prompt_4": { "overall_acc": 0.2638095238095238, "language_acc": { "Filipino": 0.21333333333333335, "Vietnamese": 0.30666666666666664, "Chinese": 0.20666666666666667, "Spanish": 0.25333333333333335, "Malay": 0.31333333333333335, "Indonesian": 0.2733333333333333, "English": 0.28 }, "consistency_score_2": 0.26031746031746034, "consistency_score_3": 0.07180952380952384, "consistency_score_4": 0.021904761904761906, "consistency_score_5": 0.006349206349206349, "consistency_score_6": 0.0009523809523809525, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.26, "Filipino,Chinese": 0.2866666666666667, "Filipino,Spanish": 0.26, "Filipino,Malay": 0.2733333333333333, "Filipino,Indonesian": 0.29333333333333333, "Filipino,English": 0.21333333333333335, "Vietnamese,Chinese": 0.21333333333333335, "Vietnamese,Spanish": 0.2733333333333333, "Vietnamese,Malay": 0.35333333333333333, "Vietnamese,Indonesian": 0.25333333333333335, "Vietnamese,English": 0.2733333333333333, "Chinese,Spanish": 0.29333333333333333, "Chinese,Malay": 0.20666666666666667, "Chinese,Indonesian": 0.24666666666666667, "Chinese,English": 0.2, "Spanish,Malay": 0.2733333333333333, "Spanish,Indonesian": 0.25333333333333335, "Spanish,English": 0.32666666666666666, "Malay,Indonesian": 0.26, "Malay,English": 0.21333333333333335, "Indonesian,English": 0.24 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.05333333333333334, "Filipino,Vietnamese,Spanish": 0.1, "Filipino,Vietnamese,Malay": 0.10666666666666667, "Filipino,Vietnamese,Indonesian": 0.06, "Filipino,Vietnamese,English": 0.06666666666666667, "Filipino,Chinese,Spanish": 0.10666666666666667, "Filipino,Chinese,Malay": 0.03333333333333333, "Filipino,Chinese,Indonesian": 0.09333333333333334, "Filipino,Chinese,English": 0.06666666666666667, "Filipino,Spanish,Malay": 0.06, "Filipino,Spanish,Indonesian": 0.07333333333333333, "Filipino,Spanish,English": 0.08666666666666667, "Filipino,Malay,Indonesian": 0.1, "Filipino,Malay,English": 0.05333333333333334, "Filipino,Indonesian,English": 0.08, "Vietnamese,Chinese,Spanish": 0.07333333333333333, "Vietnamese,Chinese,Malay": 0.05333333333333334, "Vietnamese,Chinese,Indonesian": 0.05333333333333334, "Vietnamese,Chinese,English": 0.04, "Vietnamese,Spanish,Malay": 0.10666666666666667, "Vietnamese,Spanish,Indonesian": 0.04666666666666667, "Vietnamese,Spanish,English": 0.11333333333333333, "Vietnamese,Malay,Indonesian": 0.12, "Vietnamese,Malay,English": 0.08666666666666667, "Vietnamese,Indonesian,English": 0.06, "Chinese,Spanish,Malay": 0.06666666666666667, "Chinese,Spanish,Indonesian": 0.08666666666666667, "Chinese,Spanish,English": 0.07333333333333333, "Chinese,Malay,Indonesian": 0.06, "Chinese,Malay,English": 0.006666666666666667, "Chinese,Indonesian,English": 0.06, "Spanish,Malay,Indonesian": 0.08666666666666667, "Spanish,Malay,English": 0.06, "Spanish,Indonesian,English": 0.07333333333333333, "Malay,Indonesian,English": 0.04666666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.03333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.03333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Spanish,English": 0.03333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.04666666666666667, "Filipino,Vietnamese,Malay,English": 0.03333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.04, "Filipino,Chinese,Spanish,English": 0.04, "Filipino,Chinese,Malay,Indonesian": 0.02, "Filipino,Chinese,Malay,English": 0.0, "Filipino,Chinese,Indonesian,English": 0.03333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.02, "Filipino,Spanish,Malay,English": 0.02666666666666667, "Filipino,Spanish,Indonesian,English": 0.03333333333333333, "Filipino,Malay,Indonesian,English": 0.02, "Vietnamese,Chinese,Spanish,Malay": 0.02666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.02, "Vietnamese,Chinese,Spanish,English": 0.02, "Vietnamese,Chinese,Malay,Indonesian": 0.02, "Vietnamese,Chinese,Malay,English": 0.0, "Vietnamese,Chinese,Indonesian,English": 0.006666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.04, "Vietnamese,Spanish,Malay,English": 0.04, "Vietnamese,Spanish,Indonesian,English": 0.013333333333333334, "Vietnamese,Malay,Indonesian,English": 0.02666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.04, "Chinese,Spanish,Malay,English": 0.0, "Chinese,Spanish,Indonesian,English": 0.02666666666666667, "Chinese,Malay,Indonesian,English": 0.0, "Spanish,Malay,Indonesian,English": 0.02 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.02, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Malay,Indonesian,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.013333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.0, "Filipino,Chinese,Spanish,Indonesian,English": 0.02, "Filipino,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Spanish,Malay,Indonesian,English": 0.006666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.02, "Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Vietnamese,Spanish,Malay,Indonesian,English": 0.013333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 } }, "AC3_2": 0.2620518588522548, "AC3_3": 0.11289011401425296, "AC3_4": 0.040450793636635875, "AC3_5": 0.012399977612779037, "AC3_6": 0.001897910242517148, "AC3_7": 0.0 }, "prompt_5": { "overall_acc": 0.24095238095238095, "language_acc": { "Filipino": 0.22666666666666666, "Vietnamese": 0.16666666666666666, "Chinese": 0.2866666666666667, "Spanish": 0.26666666666666666, "Malay": 0.22, "Indonesian": 0.25333333333333335, "English": 0.26666666666666666 }, "consistency_score_2": 0.25555555555555554, "consistency_score_3": 0.06019047619047618, "consistency_score_4": 0.011428571428571425, "consistency_score_5": 0.0012698412698412698, "consistency_score_6": 0.0, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.25333333333333335, "Filipino,Chinese": 0.22, "Filipino,Spanish": 0.3333333333333333, "Filipino,Malay": 0.2733333333333333, "Filipino,Indonesian": 0.24, "Filipino,English": 0.18666666666666668, "Vietnamese,Chinese": 0.3, "Vietnamese,Spanish": 0.24, "Vietnamese,Malay": 0.22666666666666666, "Vietnamese,Indonesian": 0.20666666666666667, "Vietnamese,English": 0.28, "Chinese,Spanish": 0.23333333333333334, "Chinese,Malay": 0.26, "Chinese,Indonesian": 0.26666666666666666, "Chinese,English": 0.20666666666666667, "Spanish,Malay": 0.2733333333333333, "Spanish,Indonesian": 0.26, "Spanish,English": 0.21333333333333335, "Malay,Indonesian": 0.31333333333333335, "Malay,English": 0.24666666666666667, "Indonesian,English": 0.3333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.06666666666666667, "Filipino,Vietnamese,Spanish": 0.06666666666666667, "Filipino,Vietnamese,Malay": 0.04, "Filipino,Vietnamese,Indonesian": 0.02, "Filipino,Vietnamese,English": 0.06666666666666667, "Filipino,Chinese,Spanish": 0.07333333333333333, "Filipino,Chinese,Malay": 0.05333333333333334, "Filipino,Chinese,Indonesian": 0.06, "Filipino,Chinese,English": 0.02, "Filipino,Spanish,Malay": 0.07333333333333333, "Filipino,Spanish,Indonesian": 0.06666666666666667, "Filipino,Spanish,English": 0.03333333333333333, "Filipino,Malay,Indonesian": 0.08, "Filipino,Malay,English": 0.05333333333333334, "Filipino,Indonesian,English": 0.06, "Vietnamese,Chinese,Spanish": 0.07333333333333333, "Vietnamese,Chinese,Malay": 0.06666666666666667, "Vietnamese,Chinese,Indonesian": 0.09333333333333334, "Vietnamese,Chinese,English": 0.07333333333333333, "Vietnamese,Spanish,Malay": 0.06, "Vietnamese,Spanish,Indonesian": 0.04, "Vietnamese,Spanish,English": 0.06, "Vietnamese,Malay,Indonesian": 0.04666666666666667, "Vietnamese,Malay,English": 0.06666666666666667, "Vietnamese,Indonesian,English": 0.06666666666666667, "Chinese,Spanish,Malay": 0.06, "Chinese,Spanish,Indonesian": 0.06666666666666667, "Chinese,Spanish,English": 0.03333333333333333, "Chinese,Malay,Indonesian": 0.09333333333333334, "Chinese,Malay,English": 0.04, "Chinese,Indonesian,English": 0.08, "Spanish,Malay,Indonesian": 0.05333333333333334, "Spanish,Malay,English": 0.05333333333333334, "Spanish,Indonesian,English": 0.05333333333333334, "Malay,Indonesian,English": 0.09333333333333334 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.02666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.0, "Filipino,Vietnamese,Spanish,Indonesian": 0.0, "Filipino,Vietnamese,Spanish,English": 0.013333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Malay,English": 0.02, "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.02, "Filipino,Chinese,Spanish,English": 0.006666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.02666666666666667, "Filipino,Chinese,Malay,English": 0.0, "Filipino,Chinese,Indonesian,English": 0.006666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.013333333333333334, "Filipino,Spanish,Malay,English": 0.0, "Filipino,Spanish,Indonesian,English": 0.0, "Filipino,Malay,Indonesian,English": 0.02, "Vietnamese,Chinese,Spanish,Malay": 0.02, "Vietnamese,Chinese,Spanish,Indonesian": 0.013333333333333334, "Vietnamese,Chinese,Spanish,English": 0.02, "Vietnamese,Chinese,Malay,Indonesian": 0.013333333333333334, "Vietnamese,Chinese,Malay,English": 0.013333333333333334, "Vietnamese,Chinese,Indonesian,English": 0.03333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, "Vietnamese,Spanish,Malay,English": 0.02, "Vietnamese,Spanish,Indonesian,English": 0.0, "Vietnamese,Malay,Indonesian,English": 0.006666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.02, "Chinese,Spanish,Malay,English": 0.013333333333333334, "Chinese,Spanish,Indonesian,English": 0.006666666666666667, "Chinese,Malay,Indonesian,English": 0.013333333333333334, "Spanish,Malay,Indonesian,English": 0.0 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Spanish,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.013333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.0, "Filipino,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 } }, "AC3_2": 0.24803921563631776, "AC3_3": 0.09631999032975126, "AC3_4": 0.02182210241722951, "AC3_5": 0.0025263683444603424, "AC3_6": 0.0, "AC3_7": 0.0 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.26623376623376627, "language_acc": { "Indonesian": 0.2556818181818182, "English": 0.25, "Filipino": 0.30113636363636365, "Spanish": 0.2897727272727273, "Chinese": 0.2897727272727273, "Malay": 0.21022727272727273, "Vietnamese": 0.26704545454545453 }, "consistency_score_2": 0.23457792207792205, "consistency_score_3": 0.05113636363636364, "consistency_score_4": 0.008928571428571425, "consistency_score_5": 0.0005411255411255411, "consistency_score_6": 0.0, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.22727272727272727, "Indonesian,Filipino": 0.26136363636363635, "Indonesian,Spanish": 0.22727272727272727, "Indonesian,Chinese": 0.23295454545454544, "Indonesian,Malay": 0.2784090909090909, "Indonesian,Vietnamese": 0.19318181818181818, "English,Filipino": 0.24431818181818182, "English,Spanish": 0.29545454545454547, "English,Chinese": 0.23295454545454544, "English,Malay": 0.2840909090909091, "English,Vietnamese": 0.19886363636363635, "Filipino,Spanish": 0.17613636363636365, "Filipino,Chinese": 0.2727272727272727, "Filipino,Malay": 0.2840909090909091, "Filipino,Vietnamese": 0.2159090909090909, "Spanish,Chinese": 0.1875, "Spanish,Malay": 0.21022727272727273, "Spanish,Vietnamese": 0.22727272727272727, "Chinese,Malay": 0.1875, "Chinese,Vietnamese": 0.3068181818181818, "Malay,Vietnamese": 0.18181818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.045454545454545456, "Indonesian,English,Spanish": 0.045454545454545456, "Indonesian,English,Chinese": 0.056818181818181816, "Indonesian,English,Malay": 0.056818181818181816, "Indonesian,English,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Spanish": 0.03977272727272727, "Indonesian,Filipino,Chinese": 0.07954545454545454, "Indonesian,Filipino,Malay": 0.09659090909090909, "Indonesian,Filipino,Vietnamese": 0.022727272727272728, "Indonesian,Spanish,Chinese": 0.03409090909090909, "Indonesian,Spanish,Malay": 0.03977272727272727, "Indonesian,Spanish,Vietnamese": 0.045454545454545456, "Indonesian,Chinese,Malay": 0.045454545454545456, "Indonesian,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish": 0.06818181818181818, "English,Filipino,Chinese": 0.056818181818181816, "English,Filipino,Malay": 0.10795454545454546, "English,Filipino,Vietnamese": 0.045454545454545456, "English,Spanish,Chinese": 0.06818181818181818, "English,Spanish,Malay": 0.07954545454545454, "English,Spanish,Vietnamese": 0.0625, "English,Chinese,Malay": 0.045454545454545456, "English,Chinese,Vietnamese": 0.0625, "English,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese": 0.03409090909090909, "Filipino,Spanish,Malay": 0.03409090909090909, "Filipino,Spanish,Vietnamese": 0.03977272727272727, "Filipino,Chinese,Malay": 0.06818181818181818, "Filipino,Chinese,Vietnamese": 0.0625, "Filipino,Malay,Vietnamese": 0.03409090909090909, "Spanish,Chinese,Malay": 0.011363636363636364, "Spanish,Chinese,Vietnamese": 0.0625, "Spanish,Malay,Vietnamese": 0.03977272727272727, "Chinese,Malay,Vietnamese": 0.056818181818181816 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.0, "Indonesian,English,Filipino,Chinese": 0.005681818181818182, "Indonesian,English,Filipino,Malay": 0.028409090909090908, "Indonesian,English,Filipino,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese": 0.011363636363636364, "Indonesian,English,Spanish,Malay": 0.0, "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, "Indonesian,English,Chinese,Malay": 0.0, "Indonesian,English,Chinese,Vietnamese": 0.0, "Indonesian,English,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.0, "Indonesian,Filipino,Spanish,Malay": 0.011363636363636364, "Indonesian,Filipino,Spanish,Vietnamese": 0.0, "Indonesian,Filipino,Chinese,Malay": 0.028409090909090908, "Indonesian,Filipino,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Spanish,Chinese,Malay": 0.0, "Indonesian,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Chinese": 0.022727272727272728, "English,Filipino,Spanish,Malay": 0.022727272727272728, "English,Filipino,Spanish,Vietnamese": 0.011363636363636364, "English,Filipino,Chinese,Malay": 0.011363636363636364, "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, "English,Filipino,Malay,Vietnamese": 0.011363636363636364, "English,Spanish,Chinese,Malay": 0.005681818181818182, "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, "English,Spanish,Malay,Vietnamese": 0.011363636363636364, "English,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay": 0.0, "Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Malay,Vietnamese": 0.0, "Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.0, "Indonesian,English,Filipino,Spanish,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese,Malay": 0.0, "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.0, "English,Filipino,Chinese,Malay,Vietnamese": 0.0, "English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, "AC3_2": 0.24940537577372796, "AC3_3": 0.08579400136799162, "AC3_4": 0.01727770753691517, "AC3_5": 0.00108005584638527, "AC3_6": 0.0, "AC3_7": 0.0 }, "prompt_2": { "overall_acc": 0.25243506493506496, "language_acc": { "Indonesian": 0.23295454545454544, "English": 0.2897727272727273, "Filipino": 0.2556818181818182, "Spanish": 0.2556818181818182, "Chinese": 0.26136363636363635, "Malay": 0.23295454545454544, "Vietnamese": 0.23863636363636365 }, "consistency_score_2": 0.24729437229437234, "consistency_score_3": 0.05746753246753245, "consistency_score_4": 0.012012987012987007, "consistency_score_5": 0.0018939393939393942, "consistency_score_6": 0.0, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.26136363636363635, "Indonesian,Filipino": 0.3465909090909091, "Indonesian,Spanish": 0.20454545454545456, "Indonesian,Chinese": 0.2159090909090909, "Indonesian,Malay": 0.2159090909090909, "Indonesian,Vietnamese": 0.2556818181818182, "English,Filipino": 0.2556818181818182, "English,Spanish": 0.26136363636363635, "English,Chinese": 0.23863636363636365, "English,Malay": 0.2556818181818182, "English,Vietnamese": 0.19886363636363635, "Filipino,Spanish": 0.2159090909090909, "Filipino,Chinese": 0.23295454545454544, "Filipino,Malay": 0.26136363636363635, "Filipino,Vietnamese": 0.2840909090909091, "Spanish,Chinese": 0.23863636363636365, "Spanish,Malay": 0.2727272727272727, "Spanish,Vietnamese": 0.25, "Chinese,Malay": 0.23295454545454544, "Chinese,Vietnamese": 0.23295454545454544, "Malay,Vietnamese": 0.26136363636363635 }, "3_combine": { "Indonesian,English,Filipino": 0.11931818181818182, "Indonesian,English,Spanish": 0.0625, "Indonesian,English,Chinese": 0.056818181818181816, "Indonesian,English,Malay": 0.0625, "Indonesian,English,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish": 0.045454545454545456, "Indonesian,Filipino,Chinese": 0.06818181818181818, "Indonesian,Filipino,Malay": 0.06818181818181818, "Indonesian,Filipino,Vietnamese": 0.09090909090909091, "Indonesian,Spanish,Chinese": 0.03977272727272727, "Indonesian,Spanish,Malay": 0.045454545454545456, "Indonesian,Spanish,Vietnamese": 0.03409090909090909, "Indonesian,Chinese,Malay": 0.028409090909090908, "Indonesian,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish": 0.06818181818181818, "English,Filipino,Chinese": 0.056818181818181816, "English,Filipino,Malay": 0.056818181818181816, "English,Filipino,Vietnamese": 0.0625, "English,Spanish,Chinese": 0.056818181818181816, "English,Spanish,Malay": 0.06818181818181818, "English,Spanish,Vietnamese": 0.0625, "English,Chinese,Malay": 0.045454545454545456, "English,Chinese,Vietnamese": 0.045454545454545456, "English,Malay,Vietnamese": 0.0625, "Filipino,Spanish,Chinese": 0.056818181818181816, "Filipino,Spanish,Malay": 0.05113636363636364, "Filipino,Spanish,Vietnamese": 0.05113636363636364, "Filipino,Chinese,Malay": 0.0625, "Filipino,Chinese,Vietnamese": 0.05113636363636364, "Filipino,Malay,Vietnamese": 0.06818181818181818, "Spanish,Chinese,Malay": 0.06818181818181818, "Spanish,Chinese,Vietnamese": 0.056818181818181816, "Spanish,Malay,Vietnamese": 0.07954545454545454, "Chinese,Malay,Vietnamese": 0.03409090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.028409090909090908, "Indonesian,English,Filipino,Chinese": 0.028409090909090908, "Indonesian,English,Filipino,Malay": 0.017045454545454544, "Indonesian,English,Filipino,Vietnamese": 0.017045454545454544, "Indonesian,English,Spanish,Chinese": 0.011363636363636364, "Indonesian,English,Spanish,Malay": 0.017045454545454544, "Indonesian,English,Spanish,Vietnamese": 0.0, "Indonesian,English,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Malay": 0.011363636363636364, "Indonesian,Filipino,Spanish,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Chinese,Malay": 0.0, "Indonesian,Filipino,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,Spanish,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese": 0.022727272727272728, "English,Filipino,Spanish,Malay": 0.011363636363636364, "English,Filipino,Spanish,Vietnamese": 0.0, "English,Filipino,Chinese,Malay": 0.011363636363636364, "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, "English,Filipino,Malay,Vietnamese": 0.022727272727272728, "English,Spanish,Chinese,Malay": 0.005681818181818182, "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Spanish,Malay,Vietnamese": 0.022727272727272728, "English,Chinese,Malay,Vietnamese": 0.0, "Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, "Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Malay": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese,Malay": 0.0, "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "English,Filipino,Spanish,Malay,Vietnamese": 0.0, "English,Filipino,Chinese,Malay,Vietnamese": 0.0, "English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, "AC3_2": 0.24983827753598187, "AC3_3": 0.0936218050897616, "AC3_4": 0.022934554693837875, "AC3_5": 0.0037596711784056757, "AC3_6": 0.0, "AC3_7": 0.0 }, "prompt_3": { "overall_acc": 0.2508116883116883, "language_acc": { "Indonesian": 0.24431818181818182, "English": 0.26136363636363635, "Filipino": 0.24431818181818182, "Spanish": 0.23295454545454544, "Chinese": 0.23863636363636365, "Malay": 0.26704545454545453, "Vietnamese": 0.26704545454545453 }, "consistency_score_2": 0.2440476190476191, "consistency_score_3": 0.05503246753246754, "consistency_score_4": 0.010714285714285711, "consistency_score_5": 0.001352813852813853, "consistency_score_6": 0.0, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.2159090909090909, "Indonesian,Filipino": 0.19318181818181818, "Indonesian,Spanish": 0.2556818181818182, "Indonesian,Chinese": 0.21022727272727273, "Indonesian,Malay": 0.2727272727272727, "Indonesian,Vietnamese": 0.29545454545454547, "English,Filipino": 0.24431818181818182, "English,Spanish": 0.2840909090909091, "English,Chinese": 0.17045454545454544, "English,Malay": 0.2556818181818182, "English,Vietnamese": 0.23295454545454544, "Filipino,Spanish": 0.23863636363636365, "Filipino,Chinese": 0.24431818181818182, "Filipino,Malay": 0.22727272727272727, "Filipino,Vietnamese": 0.3181818181818182, "Spanish,Chinese": 0.3068181818181818, "Spanish,Malay": 0.26136363636363635, "Spanish,Vietnamese": 0.2215909090909091, "Chinese,Malay": 0.2840909090909091, "Chinese,Vietnamese": 0.19886363636363635, "Malay,Vietnamese": 0.19318181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.017045454545454544, "Indonesian,English,Spanish": 0.05113636363636364, "Indonesian,English,Chinese": 0.03977272727272727, "Indonesian,English,Malay": 0.08522727272727272, "Indonesian,English,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish": 0.03977272727272727, "Indonesian,Filipino,Chinese": 0.022727272727272728, "Indonesian,Filipino,Malay": 0.03977272727272727, "Indonesian,Filipino,Vietnamese": 0.07386363636363637, "Indonesian,Spanish,Chinese": 0.07386363636363637, "Indonesian,Spanish,Malay": 0.07386363636363637, "Indonesian,Spanish,Vietnamese": 0.056818181818181816, "Indonesian,Chinese,Malay": 0.07386363636363637, "Indonesian,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish": 0.08522727272727272, "English,Filipino,Chinese": 0.045454545454545456, "English,Filipino,Malay": 0.06818181818181818, "English,Filipino,Vietnamese": 0.045454545454545456, "English,Spanish,Chinese": 0.056818181818181816, "English,Spanish,Malay": 0.07954545454545454, "English,Spanish,Vietnamese": 0.07386363636363637, "English,Chinese,Malay": 0.045454545454545456, "English,Chinese,Vietnamese": 0.028409090909090908, "English,Malay,Vietnamese": 0.07386363636363637, "Filipino,Spanish,Chinese": 0.0625, "Filipino,Spanish,Malay": 0.03977272727272727, "Filipino,Spanish,Vietnamese": 0.0625, "Filipino,Chinese,Malay": 0.056818181818181816, "Filipino,Chinese,Vietnamese": 0.056818181818181816, "Filipino,Malay,Vietnamese": 0.028409090909090908, "Spanish,Chinese,Malay": 0.07386363636363637, "Spanish,Chinese,Vietnamese": 0.0625, "Spanish,Malay,Vietnamese": 0.05113636363636364, "Chinese,Malay,Vietnamese": 0.03977272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.0, "Indonesian,English,Filipino,Chinese": 0.005681818181818182, "Indonesian,English,Filipino,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese": 0.011363636363636364, "Indonesian,English,Spanish,Malay": 0.022727272727272728, "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, "Indonesian,English,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Malay,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Chinese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Malay": 0.011363636363636364, "Indonesian,Filipino,Spanish,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Chinese,Malay": 0.017045454545454544, "Indonesian,Filipino,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,Spanish,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Chinese": 0.022727272727272728, "English,Filipino,Spanish,Malay": 0.017045454545454544, "English,Filipino,Spanish,Vietnamese": 0.017045454545454544, "English,Filipino,Chinese,Malay": 0.005681818181818182, "English,Filipino,Chinese,Vietnamese": 0.005681818181818182, "English,Filipino,Malay,Vietnamese": 0.011363636363636364, "English,Spanish,Chinese,Malay": 0.011363636363636364, "English,Spanish,Chinese,Vietnamese": 0.005681818181818182, "English,Spanish,Malay,Vietnamese": 0.03409090909090909, "English,Chinese,Malay,Vietnamese": 0.005681818181818182, "Filipino,Spanish,Chinese,Malay": 0.005681818181818182, "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Filipino,Spanish,Malay,Vietnamese": 0.0, "Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.0, "Indonesian,English,Filipino,Spanish,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese,Malay": 0.0, "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.0, "English,Filipino,Chinese,Malay,Vietnamese": 0.0, "English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, "AC3_2": 0.24738342570959695, "AC3_3": 0.09026025722915307, "AC3_4": 0.020550678364049425, "AC3_5": 0.0026911125344659314, "AC3_6": 0.0, "AC3_7": 0.0 }, "prompt_4": { "overall_acc": 0.2394480519480519, "language_acc": { "Indonesian": 0.2556818181818182, "English": 0.23295454545454544, "Filipino": 0.26136363636363635, "Spanish": 0.21022727272727273, "Chinese": 0.2556818181818182, "Malay": 0.26136363636363635, "Vietnamese": 0.19886363636363635 }, "consistency_score_2": 0.25676406926406925, "consistency_score_3": 0.0681818181818182, "consistency_score_4": 0.021753246753246754, "consistency_score_5": 0.009740259740259737, "consistency_score_6": 0.006493506493506494, "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.2727272727272727, "Indonesian,Filipino": 0.2159090909090909, "Indonesian,Spanish": 0.2215909090909091, "Indonesian,Chinese": 0.23295454545454544, "Indonesian,Malay": 0.25, "Indonesian,Vietnamese": 0.29545454545454547, "English,Filipino": 0.2159090909090909, "English,Spanish": 0.2556818181818182, "English,Chinese": 0.26136363636363635, "English,Malay": 0.23863636363636365, "English,Vietnamese": 0.3181818181818182, "Filipino,Spanish": 0.26704545454545453, "Filipino,Chinese": 0.32386363636363635, "Filipino,Malay": 0.2556818181818182, "Filipino,Vietnamese": 0.26704545454545453, "Spanish,Chinese": 0.26136363636363635, "Spanish,Malay": 0.25, "Spanish,Vietnamese": 0.24431818181818182, "Chinese,Malay": 0.25, "Chinese,Vietnamese": 0.1875, "Malay,Vietnamese": 0.3068181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.045454545454545456, "Indonesian,English,Spanish": 0.056818181818181816, "Indonesian,English,Chinese": 0.0625, "Indonesian,English,Malay": 0.056818181818181816, "Indonesian,English,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish": 0.045454545454545456, "Indonesian,Filipino,Chinese": 0.06818181818181818, "Indonesian,Filipino,Malay": 0.056818181818181816, "Indonesian,Filipino,Vietnamese": 0.08522727272727272, "Indonesian,Spanish,Chinese": 0.0625, "Indonesian,Spanish,Malay": 0.05113636363636364, "Indonesian,Spanish,Vietnamese": 0.056818181818181816, "Indonesian,Chinese,Malay": 0.07954545454545454, "Indonesian,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish": 0.056818181818181816, "English,Filipino,Chinese": 0.08522727272727272, "English,Filipino,Malay": 0.056818181818181816, "English,Filipino,Vietnamese": 0.09659090909090909, "English,Spanish,Chinese": 0.07954545454545454, "English,Spanish,Malay": 0.07954545454545454, "English,Spanish,Vietnamese": 0.07386363636363637, "English,Chinese,Malay": 0.0625, "English,Chinese,Vietnamese": 0.045454545454545456, "English,Malay,Vietnamese": 0.07954545454545454, "Filipino,Spanish,Chinese": 0.09090909090909091, "Filipino,Spanish,Malay": 0.07386363636363637, "Filipino,Spanish,Vietnamese": 0.06818181818181818, "Filipino,Chinese,Malay": 0.07954545454545454, "Filipino,Chinese,Vietnamese": 0.07386363636363637, "Filipino,Malay,Vietnamese": 0.09659090909090909, "Spanish,Chinese,Malay": 0.06818181818181818, "Spanish,Chinese,Vietnamese": 0.045454545454545456, "Spanish,Malay,Vietnamese": 0.05113636363636364, "Chinese,Malay,Vietnamese": 0.06818181818181818 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.011363636363636364, "Indonesian,English,Filipino,Chinese": 0.011363636363636364, "Indonesian,English,Filipino,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Vietnamese": 0.03977272727272727, "Indonesian,English,Spanish,Chinese": 0.017045454545454544, "Indonesian,English,Spanish,Malay": 0.022727272727272728, "Indonesian,English,Spanish,Vietnamese": 0.022727272727272728, "Indonesian,English,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Malay": 0.017045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Chinese,Malay": 0.022727272727272728, "Indonesian,Filipino,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Malay,Vietnamese": 0.028409090909090908, "Indonesian,Spanish,Chinese,Malay": 0.022727272727272728, "Indonesian,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,Spanish,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Chinese": 0.022727272727272728, "English,Filipino,Spanish,Malay": 0.03409090909090909, "English,Filipino,Spanish,Vietnamese": 0.022727272727272728, "English,Filipino,Chinese,Malay": 0.022727272727272728, "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Malay,Vietnamese": 0.028409090909090908, "English,Spanish,Chinese,Malay": 0.03409090909090909, "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Spanish,Malay,Vietnamese": 0.022727272727272728, "English,Chinese,Malay,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Filipino,Spanish,Malay,Vietnamese": 0.022727272727272728, "Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "English,Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, "English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, "AC3_2": 0.24780392721145814, "AC3_3": 0.10614056125117736, "AC3_4": 0.03988320565413914, "AC3_5": 0.018719065942824287, "AC3_6": 0.012644121549871535, "AC3_7": 0.011100240814255554 }, "prompt_5": { "overall_acc": 0.2605519480519481, "language_acc": { "Indonesian": 0.2215909090909091, "English": 0.2215909090909091, "Filipino": 0.30113636363636365, "Spanish": 0.23863636363636365, "Chinese": 0.2727272727272727, "Malay": 0.2727272727272727, "Vietnamese": 0.29545454545454547 }, "consistency_score_2": 0.25162337662337664, "consistency_score_3": 0.06185064935064933, "consistency_score_4": 0.01233766233766233, "consistency_score_5": 0.0010822510822510823, "consistency_score_6": 0.0, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.23863636363636365, "Indonesian,Filipino": 0.2784090909090909, "Indonesian,Spanish": 0.3068181818181818, "Indonesian,Chinese": 0.26136363636363635, "Indonesian,Malay": 0.2215909090909091, "Indonesian,Vietnamese": 0.2159090909090909, "English,Filipino": 0.23863636363636365, "English,Spanish": 0.2897727272727273, "English,Chinese": 0.2556818181818182, "English,Malay": 0.2784090909090909, "English,Vietnamese": 0.23863636363636365, "Filipino,Spanish": 0.2897727272727273, "Filipino,Chinese": 0.20454545454545456, "Filipino,Malay": 0.2215909090909091, "Filipino,Vietnamese": 0.22727272727272727, "Spanish,Chinese": 0.22727272727272727, "Spanish,Malay": 0.23295454545454544, "Spanish,Vietnamese": 0.2897727272727273, "Chinese,Malay": 0.23863636363636365, "Chinese,Vietnamese": 0.26136363636363635, "Malay,Vietnamese": 0.26704545454545453 }, "3_combine": { "Indonesian,English,Filipino": 0.0625, "Indonesian,English,Spanish": 0.07386363636363637, "Indonesian,English,Chinese": 0.06818181818181818, "Indonesian,English,Malay": 0.07386363636363637, "Indonesian,English,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish": 0.10227272727272728, "Indonesian,Filipino,Chinese": 0.0625, "Indonesian,Filipino,Malay": 0.05113636363636364, "Indonesian,Filipino,Vietnamese": 0.045454545454545456, "Indonesian,Spanish,Chinese": 0.05113636363636364, "Indonesian,Spanish,Malay": 0.056818181818181816, "Indonesian,Spanish,Vietnamese": 0.056818181818181816, "Indonesian,Chinese,Malay": 0.07386363636363637, "Indonesian,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish": 0.07954545454545454, "English,Filipino,Chinese": 0.03977272727272727, "English,Filipino,Malay": 0.0625, "English,Filipino,Vietnamese": 0.0625, "English,Spanish,Chinese": 0.0625, "English,Spanish,Malay": 0.06818181818181818, "English,Spanish,Vietnamese": 0.06818181818181818, "English,Chinese,Malay": 0.045454545454545456, "English,Chinese,Vietnamese": 0.06818181818181818, "English,Malay,Vietnamese": 0.08522727272727272, "Filipino,Spanish,Chinese": 0.07386363636363637, "Filipino,Spanish,Malay": 0.056818181818181816, "Filipino,Spanish,Vietnamese": 0.06818181818181818, "Filipino,Chinese,Malay": 0.045454545454545456, "Filipino,Chinese,Vietnamese": 0.03977272727272727, "Filipino,Malay,Vietnamese": 0.05113636363636364, "Spanish,Chinese,Malay": 0.05113636363636364, "Spanish,Chinese,Vietnamese": 0.07386363636363637, "Spanish,Malay,Vietnamese": 0.07386363636363637, "Chinese,Malay,Vietnamese": 0.0625 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.017045454545454544, "Indonesian,English,Filipino,Chinese": 0.011363636363636364, "Indonesian,English,Filipino,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Chinese": 0.017045454545454544, "Indonesian,English,Spanish,Malay": 0.011363636363636364, "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, "Indonesian,English,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Chinese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Malay": 0.017045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Chinese,Malay": 0.011363636363636364, "Indonesian,Filipino,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Malay,Vietnamese": 0.0, "Indonesian,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.017045454545454544, "Indonesian,Chinese,Malay,Vietnamese": 0.011363636363636364, "English,Filipino,Spanish,Chinese": 0.011363636363636364, "English,Filipino,Spanish,Malay": 0.017045454545454544, "English,Filipino,Spanish,Vietnamese": 0.011363636363636364, "English,Filipino,Chinese,Malay": 0.0, "English,Filipino,Chinese,Vietnamese": 0.005681818181818182, "English,Filipino,Malay,Vietnamese": 0.022727272727272728, "English,Spanish,Chinese,Malay": 0.005681818181818182, "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Spanish,Malay,Vietnamese": 0.011363636363636364, "English,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay": 0.011363636363636364, "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.0, "Indonesian,English,Filipino,Spanish,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.0, "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "English,Filipino,Spanish,Malay,Vietnamese": 0.0, "English,Filipino,Chinese,Malay,Vietnamese": 0.0, "English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, "AC3_2": 0.2560098379725315, "AC3_3": 0.09997008275629547, "AC3_4": 0.023559724026997794, "AC3_5": 0.0021555486904021653, "AC3_6": 0.0, "AC3_7": 0.0 } }, "sg_eval": { "prompt_1": { "accuracy": 0.27184466019417475 }, "prompt_2": { "accuracy": 0.2912621359223301 }, "prompt_3": { "accuracy": 0.32038834951456313 }, "prompt_4": { "accuracy": 0.24271844660194175 }, "prompt_5": { "accuracy": 0.21359223300970873 } }, "cn_eval": { "prompt_1": { "accuracy": 0.29523809523809524 }, "prompt_2": { "accuracy": 0.2 }, "prompt_3": { "accuracy": 0.2571428571428571 }, "prompt_4": { "accuracy": 0.2761904761904762 }, "prompt_5": { "accuracy": 0.21904761904761905 } }, "us_eval": { "prompt_1": { "accuracy": 0.2897196261682243 }, "prompt_2": { "accuracy": 0.2523364485981308 }, "prompt_3": { "accuracy": 0.2523364485981308 }, "prompt_4": { "accuracy": 0.19626168224299065 }, "prompt_5": { "accuracy": 0.19626168224299065 } }, "ph_eval": { "prompt_1": { "accuracy": 0.22, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.3, "history": 0.06666666666666667, "literature": 0.3, "politics": 0.1, "culture": 0.1, "film": 0.3, "law": 0.3, "geography": 0.4 } }, "prompt_2": { "accuracy": 0.25, "category_acc": { "brand": 0.5, "demographics": 0.4, "biology": 0.3, "history": 0.2, "literature": 0.1, "politics": 0.2, "culture": 0.1, "film": 0.4, "law": 0.1, "geography": 0.3 } }, "prompt_3": { "accuracy": 0.19, "category_acc": { "brand": 0.0, "demographics": 0.4, "biology": 0.2, "history": 0.0, "literature": 0.0, "politics": 0.8, "culture": 0.1, "film": 0.2, "law": 0.2, "geography": 0.2 } }, "prompt_4": { "accuracy": 0.26, "category_acc": { "brand": 0.1, "demographics": 0.4, "biology": 0.4, "history": 0.3333333333333333, "literature": 0.1, "politics": 0.1, "culture": 0.5, "film": 0.3, "law": 0.2, "geography": 0.2 } }, "prompt_5": { "accuracy": 0.25, "category_acc": { "brand": 0.1, "demographics": 0.4, "biology": 0.3, "history": 0.2, "literature": 0.3, "politics": 0.1, "culture": 0.4, "film": 0.1, "law": 0.4, "geography": 0.3 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.00845529439051568 }, "prompt_2": { "bleu_score": 0.00845589251198986 }, "prompt_3": { "bleu_score": 0.00888315083476906 }, "prompt_4": { "bleu_score": 0.0086399802884862 }, "prompt_5": { "bleu_score": 0.008537367494631853 } }, "indommlu": { "prompt_1": { "accuracy": 0.23880098804993657, "category_acc": { "History": 0.1746987951807229, "Geography": 0.22040816326530613, "Lampungic": 0.2789115646258503, "Social science": 0.24540901502504173, "Balinese": 0.23142250530785563, "Makassarese": 0.25268817204301075, "Banjarese": 0.2569444444444444, "Chemistry": 0.22481751824817517, "Biology": 0.22603550295857988, "Science": 0.2528379772961816, "Christian religion": 0.24378109452736318, "Art": 0.2579034941763727, "Islam religion": 0.2119487908961593, "Hindu religion": 0.24, "Madurese": 0.2983050847457627, "Sport": 0.2972972972972973, "Indonesian language": 0.25062266500622665, "Physics": 0.21818181818181817, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.26605504587155965, "Sociology": 0.18548387096774194, "Economy": 0.2192622950819672, "Sundanese": 0.27052722558340536, "Javanese": 0.24294354838709678, "Civic education": 0.2117310443490701 } }, "prompt_2": { "accuracy": 0.2410708324988317, "category_acc": { "History": 0.1827309236947791, "Geography": 0.18571428571428572, "Lampungic": 0.2789115646258503, "Social science": 0.27378964941569284, "Balinese": 0.27176220806794055, "Makassarese": 0.26344086021505375, "Banjarese": 0.2777777777777778, "Chemistry": 0.2116788321167883, "Biology": 0.2, "Science": 0.25077399380804954, "Christian religion": 0.24875621890547264, "Art": 0.24459234608985025, "Islam religion": 0.2546230440967283, "Hindu religion": 0.28, "Madurese": 0.25084745762711863, "Sport": 0.25675675675675674, "Indonesian language": 0.24283935242839352, "Physics": 0.1797979797979798, "Minangkabau culture": 0.22613065326633167, "Dayak language": 0.1834862385321101, "Sociology": 0.20766129032258066, "Economy": 0.2069672131147541, "Sundanese": 0.28262748487467587, "Javanese": 0.2852822580645161, "Civic education": 0.24606580829756797 } }, "prompt_3": { "accuracy": 0.2364643834701916, "category_acc": { "History": 0.19678714859437751, "Geography": 0.20612244897959184, "Lampungic": 0.24489795918367346, "Social science": 0.2838063439065108, "Balinese": 0.2760084925690021, "Makassarese": 0.23118279569892472, "Banjarese": 0.2986111111111111, "Chemistry": 0.1927007299270073, "Biology": 0.22248520710059172, "Science": 0.26831785345717235, "Christian religion": 0.20398009950248755, "Art": 0.2512479201331115, "Islam religion": 0.20910384068278806, "Hindu religion": 0.25333333333333335, "Madurese": 0.2135593220338983, "Sport": 0.22297297297297297, "Indonesian language": 0.24470734744707348, "Physics": 0.2202020202020202, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.1743119266055046, "Sociology": 0.21975806451612903, "Economy": 0.2192622950819672, "Sundanese": 0.23681936041486604, "Javanese": 0.26411290322580644, "Civic education": 0.2217453505007153 } }, "prompt_4": { "accuracy": 0.23559650176914346, "category_acc": { "History": 0.19678714859437751, "Geography": 0.22653061224489796, "Lampungic": 0.24489795918367346, "Social science": 0.2671118530884808, "Balinese": 0.25477707006369427, "Makassarese": 0.22580645161290322, "Banjarese": 0.2013888888888889, "Chemistry": 0.1897810218978102, "Biology": 0.1881656804733728, "Science": 0.25386996904024767, "Christian religion": 0.23880597014925373, "Art": 0.24126455906821964, "Islam religion": 0.2802275960170697, "Hindu religion": 0.2866666666666667, "Madurese": 0.19661016949152543, "Sport": 0.18243243243243243, "Indonesian language": 0.24533001245330013, "Physics": 0.18585858585858586, "Minangkabau culture": 0.24623115577889448, "Dayak language": 0.25688073394495414, "Sociology": 0.21975806451612903, "Economy": 0.19467213114754098, "Sundanese": 0.26534140017286084, "Javanese": 0.24798387096774194, "Civic education": 0.2374821173104435 } }, "prompt_5": { "accuracy": 0.23145737365645236, "category_acc": { "History": 0.21686746987951808, "Geography": 0.21836734693877552, "Lampungic": 0.2653061224489796, "Social science": 0.2888146911519199, "Balinese": 0.24203821656050956, "Makassarese": 0.24731182795698925, "Banjarese": 0.2708333333333333, "Chemistry": 0.21313868613138687, "Biology": 0.2, "Science": 0.2218782249742002, "Christian religion": 0.21393034825870647, "Art": 0.2113144758735441, "Islam religion": 0.22617354196301565, "Hindu religion": 0.2733333333333333, "Madurese": 0.2542372881355932, "Sport": 0.21621621621621623, "Indonesian language": 0.24221668742216687, "Physics": 0.20404040404040405, "Minangkabau culture": 0.22110552763819097, "Dayak language": 0.23853211009174313, "Sociology": 0.22379032258064516, "Economy": 0.20081967213114754, "Sundanese": 0.2644770959377701, "Javanese": 0.22681451612903225, "Civic education": 0.20743919885550788 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.0094989992928008 }, "prompt_2": { "bleu_score": 0.00948411159412573 }, "prompt_3": { "bleu_score": 0.009230011413355381 }, "prompt_4": { "bleu_score": 0.009319569974615713 }, "prompt_5": { "bleu_score": 0.009396177509831877 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.009454892363036137 }, "prompt_2": { "bleu_score": 0.00936148871734506 }, "prompt_3": { "bleu_score": 0.009360333481711986 }, "prompt_4": { "bleu_score": 0.009209909575947904 }, "prompt_5": { "bleu_score": 0.009456497491476989 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.009551814087188693 }, "prompt_2": { "bleu_score": 0.009370587804251214 }, "prompt_3": { "bleu_score": 0.009307087651375135 }, "prompt_4": { "bleu_score": 0.009352766278754754 }, "prompt_5": { "bleu_score": 0.00934145442627295 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.009286655895261653 }, "prompt_2": { "bleu_score": 0.009381494437991243 }, "prompt_3": { "bleu_score": 0.009176759653251399 }, "prompt_4": { "bleu_score": 0.009414395334233324 }, "prompt_5": { "bleu_score": 0.009602584340295471 } }, "mmlu": { "prompt_1": { "accuracy": 0.2718786464410735 }, "prompt_2": { "accuracy": 0.24504084014002334 }, "prompt_3": { "accuracy": 0.22753792298716452 }, "prompt_4": { "accuracy": 0.24154025670945156 }, "prompt_5": { "accuracy": 0.2322053675612602 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.2474794422595638, "category_acc": { "high_school_european_history": 0.2926829268292683, "business_ethics": 0.25252525252525254, "clinical_knowledge": 0.23484848484848486, "medical_genetics": 0.24242424242424243, "high_school_us_history": 0.21674876847290642, "high_school_physics": 0.26666666666666666, "high_school_world_history": 0.24152542372881355, "virology": 0.18787878787878787, "high_school_microeconomics": 0.22784810126582278, "econometrics": 0.20353982300884957, "college_computer_science": 0.23232323232323232, "high_school_biology": 0.2168284789644013, "abstract_algebra": 0.26262626262626265, "professional_accounting": 0.2313167259786477, "philosophy": 0.2709677419354839, "professional_medicine": 0.1992619926199262, "nutrition": 0.24918032786885247, "global_facts": 0.23232323232323232, "machine_learning": 0.2702702702702703, "security_studies": 0.3319672131147541, "public_relations": 0.1834862385321101, "professional_psychology": 0.24549918166939444, "prehistory": 0.25386996904024767, "anatomy": 0.26119402985074625, "human_sexuality": 0.3, "college_medicine": 0.27906976744186046, "high_school_government_and_politics": 0.2552083333333333, "college_chemistry": 0.26262626262626265, "logical_fallacies": 0.2345679012345679, "high_school_geography": 0.2182741116751269, "elementary_mathematics": 0.2413793103448276, "human_aging": 0.25225225225225223, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.25183823529411764, "formal_logic": 0.328, "high_school_statistics": 0.2558139534883721, "international_law": 0.25833333333333336, "high_school_mathematics": 0.20446096654275092, "high_school_computer_science": 0.20202020202020202, "conceptual_physics": 0.2264957264957265, "miscellaneous": 0.24808184143222506, "high_school_chemistry": 0.2524752475247525, "marketing": 0.2446351931330472, "professional_law": 0.24787997390737115, "management": 0.3137254901960784, "college_physics": 0.19801980198019803, "jurisprudence": 0.2616822429906542, "world_religions": 0.27058823529411763, "sociology": 0.255, "us_foreign_policy": 0.2727272727272727, "high_school_macroeconomics": 0.2442159383033419, "computer_security": 0.24242424242424243, "moral_scenarios": 0.2539149888143177, "moral_disputes": 0.2608695652173913, "electrical_engineering": 0.2638888888888889, "astronomy": 0.2119205298013245, "college_biology": 0.26573426573426573 } }, "prompt_2": { "accuracy": 0.2531283518055059, "category_acc": { "high_school_european_history": 0.2865853658536585, "business_ethics": 0.30303030303030304, "clinical_knowledge": 0.26515151515151514, "medical_genetics": 0.29292929292929293, "high_school_us_history": 0.21674876847290642, "high_school_physics": 0.21333333333333335, "high_school_world_history": 0.2966101694915254, "virology": 0.23030303030303031, "high_school_microeconomics": 0.3206751054852321, "econometrics": 0.2743362831858407, "college_computer_science": 0.32323232323232326, "high_school_biology": 0.23948220064724918, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.2846975088967972, "philosophy": 0.23548387096774193, "professional_medicine": 0.2730627306273063, "nutrition": 0.23934426229508196, "global_facts": 0.23232323232323232, "machine_learning": 0.24324324324324326, "security_studies": 0.26229508196721313, "public_relations": 0.3211009174311927, "professional_psychology": 0.23731587561374795, "prehistory": 0.26006191950464397, "anatomy": 0.20149253731343283, "human_sexuality": 0.27692307692307694, "college_medicine": 0.23837209302325582, "high_school_government_and_politics": 0.22916666666666666, "college_chemistry": 0.2222222222222222, "logical_fallacies": 0.21604938271604937, "high_school_geography": 0.233502538071066, "elementary_mathematics": 0.22281167108753316, "human_aging": 0.25675675675675674, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.24080882352941177, "formal_logic": 0.24, "high_school_statistics": 0.24186046511627907, "international_law": 0.25833333333333336, "high_school_mathematics": 0.24535315985130113, "high_school_computer_science": 0.32323232323232326, "conceptual_physics": 0.23504273504273504, "miscellaneous": 0.27877237851662406, "high_school_chemistry": 0.25742574257425743, "marketing": 0.21030042918454936, "professional_law": 0.2563600782778865, "management": 0.22549019607843138, "college_physics": 0.26732673267326734, "jurisprudence": 0.308411214953271, "world_religions": 0.2529411764705882, "sociology": 0.255, "us_foreign_policy": 0.1919191919191919, "high_school_macroeconomics": 0.2442159383033419, "computer_security": 0.2222222222222222, "moral_scenarios": 0.2360178970917226, "moral_disputes": 0.26666666666666666, "electrical_engineering": 0.2222222222222222, "astronomy": 0.2781456953642384, "college_biology": 0.2867132867132867 } }, "prompt_3": { "accuracy": 0.24669288523417948, "category_acc": { "high_school_european_history": 0.24390243902439024, "business_ethics": 0.2222222222222222, "clinical_knowledge": 0.26136363636363635, "medical_genetics": 0.25252525252525254, "high_school_us_history": 0.26108374384236455, "high_school_physics": 0.22, "high_school_world_history": 0.3135593220338983, "virology": 0.2787878787878788, "high_school_microeconomics": 0.2320675105485232, "econometrics": 0.22123893805309736, "college_computer_science": 0.21212121212121213, "high_school_biology": 0.20388349514563106, "abstract_algebra": 0.18181818181818182, "professional_accounting": 0.20640569395017794, "philosophy": 0.2129032258064516, "professional_medicine": 0.2656826568265683, "nutrition": 0.26229508196721313, "global_facts": 0.26262626262626265, "machine_learning": 0.24324324324324326, "security_studies": 0.21721311475409835, "public_relations": 0.29357798165137616, "professional_psychology": 0.2225859247135843, "prehistory": 0.3126934984520124, "anatomy": 0.27611940298507465, "human_sexuality": 0.2692307692307692, "college_medicine": 0.23255813953488372, "high_school_government_and_politics": 0.2604166666666667, "college_chemistry": 0.25252525252525254, "logical_fallacies": 0.20987654320987653, "high_school_geography": 0.2131979695431472, "elementary_mathematics": 0.23076923076923078, "human_aging": 0.2927927927927928, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.25, "formal_logic": 0.2, "high_school_statistics": 0.2186046511627907, "international_law": 0.2833333333333333, "high_school_mathematics": 0.26394052044609667, "high_school_computer_science": 0.2828282828282828, "conceptual_physics": 0.23076923076923078, "miscellaneous": 0.26854219948849106, "high_school_chemistry": 0.26732673267326734, "marketing": 0.26180257510729615, "professional_law": 0.24266144814090018, "management": 0.19607843137254902, "college_physics": 0.25742574257425743, "jurisprudence": 0.27102803738317754, "world_religions": 0.22941176470588234, "sociology": 0.25, "us_foreign_policy": 0.23232323232323232, "high_school_macroeconomics": 0.2442159383033419, "computer_security": 0.29292929292929293, "moral_scenarios": 0.24049217002237136, "moral_disputes": 0.2463768115942029, "electrical_engineering": 0.19444444444444445, "astronomy": 0.24503311258278146, "college_biology": 0.32167832167832167 } }, "prompt_4": { "accuracy": 0.2474794422595638, "category_acc": { "high_school_european_history": 0.23170731707317074, "business_ethics": 0.2222222222222222, "clinical_knowledge": 0.2765151515151515, "medical_genetics": 0.1919191919191919, "high_school_us_history": 0.18226600985221675, "high_school_physics": 0.22, "high_school_world_history": 0.24152542372881355, "virology": 0.3090909090909091, "high_school_microeconomics": 0.22362869198312235, "econometrics": 0.24778761061946902, "college_computer_science": 0.21212121212121213, "high_school_biology": 0.24919093851132687, "abstract_algebra": 0.32323232323232326, "professional_accounting": 0.2669039145907473, "philosophy": 0.2645161290322581, "professional_medicine": 0.2693726937269373, "nutrition": 0.24262295081967214, "global_facts": 0.1919191919191919, "machine_learning": 0.18018018018018017, "security_studies": 0.25, "public_relations": 0.1834862385321101, "professional_psychology": 0.25859247135842883, "prehistory": 0.23219814241486067, "anatomy": 0.2537313432835821, "human_sexuality": 0.25384615384615383, "college_medicine": 0.20930232558139536, "high_school_government_and_politics": 0.234375, "college_chemistry": 0.32323232323232326, "logical_fallacies": 0.2777777777777778, "high_school_geography": 0.2639593908629442, "elementary_mathematics": 0.2440318302387268, "human_aging": 0.25675675675675674, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.2775735294117647, "formal_logic": 0.264, "high_school_statistics": 0.19069767441860466, "international_law": 0.21666666666666667, "high_school_mathematics": 0.26765799256505574, "high_school_computer_science": 0.30303030303030304, "conceptual_physics": 0.21794871794871795, "miscellaneous": 0.24808184143222506, "high_school_chemistry": 0.2722772277227723, "marketing": 0.2532188841201717, "professional_law": 0.23352902804957598, "management": 0.3137254901960784, "college_physics": 0.37623762376237624, "jurisprudence": 0.17757009345794392, "world_religions": 0.27058823529411763, "sociology": 0.26, "us_foreign_policy": 0.2222222222222222, "high_school_macroeconomics": 0.2519280205655527, "computer_security": 0.2828282828282828, "moral_scenarios": 0.24161073825503357, "moral_disputes": 0.25507246376811593, "electrical_engineering": 0.20833333333333334, "astronomy": 0.2119205298013245, "college_biology": 0.2727272727272727 } }, "prompt_5": { "accuracy": 0.24461923489452986, "category_acc": { "high_school_european_history": 0.23170731707317074, "business_ethics": 0.23232323232323232, "clinical_knowledge": 0.21212121212121213, "medical_genetics": 0.3333333333333333, "high_school_us_history": 0.22660098522167488, "high_school_physics": 0.19333333333333333, "high_school_world_history": 0.2457627118644068, "virology": 0.21818181818181817, "high_school_microeconomics": 0.24472573839662448, "econometrics": 0.26548672566371684, "college_computer_science": 0.25252525252525254, "high_school_biology": 0.2588996763754045, "abstract_algebra": 0.1919191919191919, "professional_accounting": 0.20284697508896798, "philosophy": 0.2709677419354839, "professional_medicine": 0.23985239852398524, "nutrition": 0.25245901639344265, "global_facts": 0.2828282828282828, "machine_learning": 0.26126126126126126, "security_studies": 0.2459016393442623, "public_relations": 0.25688073394495414, "professional_psychology": 0.2569558101472995, "prehistory": 0.22291021671826625, "anatomy": 0.21641791044776118, "human_sexuality": 0.26153846153846155, "college_medicine": 0.22674418604651161, "high_school_government_and_politics": 0.2552083333333333, "college_chemistry": 0.21212121212121213, "logical_fallacies": 0.2654320987654321, "high_school_geography": 0.28426395939086296, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.23873873873873874, "college_mathematics": 0.24242424242424243, "high_school_psychology": 0.25183823529411764, "formal_logic": 0.264, "high_school_statistics": 0.24186046511627907, "international_law": 0.20833333333333334, "high_school_mathematics": 0.21189591078066913, "high_school_computer_science": 0.2222222222222222, "conceptual_physics": 0.2564102564102564, "miscellaneous": 0.22506393861892582, "high_school_chemistry": 0.20297029702970298, "marketing": 0.21888412017167383, "professional_law": 0.2517938682322244, "management": 0.23529411764705882, "college_physics": 0.31683168316831684, "jurisprudence": 0.29906542056074764, "world_religions": 0.2823529411764706, "sociology": 0.24, "us_foreign_policy": 0.2222222222222222, "high_school_macroeconomics": 0.2442159383033419, "computer_security": 0.23232323232323232, "moral_scenarios": 0.24720357941834453, "moral_disputes": 0.2579710144927536, "electrical_engineering": 0.2152777777777778, "astronomy": 0.31788079470198677, "college_biology": 0.22377622377622378 } } }, "c_eval": { "prompt_1": { "accuracy": 0.2585438335809807 }, "prompt_2": { "accuracy": 0.25037147102526003 }, "prompt_3": { "accuracy": 0.2652303120356612 }, "prompt_4": { "accuracy": 0.2540861812778603 }, "prompt_5": { "accuracy": 0.24888558692421991 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.24097135740971357, "category_acc": { "computer_network": 0.16666666666666666, "operating_system": 0.041666666666666664, "computer_architecture": 0.3076923076923077, "college_programming": 0.23809523809523808, "college_physics": 0.375, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.27586206896551724, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.125, "high_school_biology": 0.25, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.11538461538461539, "middle_school_physics": 0.20833333333333334, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.31666666666666665, "business_administration": 0.15789473684210525, "marxism": 0.25, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.17647058823529413, "teacher_qualification": 0.1836734693877551, "high_school_politics": 0.4166666666666667, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.375, "logic": 0.25925925925925924, "law": 0.3448275862068966, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.14285714285714285, "high_school_chinese": 0.16666666666666666, "high_school_history": 0.24, "middle_school_history": 0.1111111111111111, "civil_servant": 0.17307692307692307, "sports_science": 0.20833333333333334, "plant_protection": 0.14814814814814814, "basic_medicine": 0.16666666666666666, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.23529411764705882, "accountant": 0.25925925925925924, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.3055555555555556, "tax_accountant": 0.24074074074074073, "physician": 0.25925925925925924 } }, "prompt_2": { "accuracy": 0.2696139476961395, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.3333333333333333, "computer_architecture": 0.34615384615384615, "college_programming": 0.21428571428571427, "college_physics": 0.16666666666666666, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.16666666666666666, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.3076923076923077, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.14285714285714285, "college_economics": 0.25, "business_administration": 0.23684210526315788, "marxism": 0.125, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.35294117647058826, "teacher_qualification": 0.24489795918367346, "high_school_politics": 0.375, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.20833333333333334, "logic": 0.3333333333333333, "law": 0.20689655172413793, "chinese_language_and_literature": 0.14285714285714285, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.4117647058823529, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.28, "middle_school_history": 0.1111111111111111, "civil_servant": 0.36538461538461536, "sports_science": 0.20833333333333334, "plant_protection": 0.3333333333333333, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.18518518518518517, "urban_and_rural_planner": 0.1568627450980392, "accountant": 0.37037037037037035, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.3333333333333333, "tax_accountant": 0.2962962962962963, "physician": 0.2777777777777778 } }, "prompt_3": { "accuracy": 0.2465753424657534, "category_acc": { "computer_network": 0.125, "operating_system": 0.25, "computer_architecture": 0.23076923076923078, "college_programming": 0.30952380952380953, "college_physics": 0.25, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.19047619047619047, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.25, "high_school_chemistry": 0.08333333333333333, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.375, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.18333333333333332, "business_administration": 0.3157894736842105, "marxism": 0.125, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.14285714285714285, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.25, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.17647058823529413, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.18518518518518517, "law": 0.2413793103448276, "chinese_language_and_literature": 0.42857142857142855, "art_studies": 0.23684210526315788, "professional_tour_guide": 0.20588235294117646, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.375, "high_school_history": 0.2, "middle_school_history": 0.2222222222222222, "civil_servant": 0.25, "sports_science": 0.375, "plant_protection": 0.2962962962962963, "basic_medicine": 0.25, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.23529411764705882, "accountant": 0.2037037037037037, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.1388888888888889, "tax_accountant": 0.37037037037037035, "physician": 0.2037037037037037 } }, "prompt_4": { "accuracy": 0.2546699875466999, "category_acc": { "computer_network": 0.375, "operating_system": 0.16666666666666666, "computer_architecture": 0.3076923076923077, "college_programming": 0.30952380952380953, "college_physics": 0.2916666666666667, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.375, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.09523809523809523, "metrology_engineer": 0.3103448275862069, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.375, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.07692307692307693, "middle_school_physics": 0.125, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.25, "college_economics": 0.21666666666666667, "business_administration": 0.2894736842105263, "marxism": 0.20833333333333334, "mao_zedong_thought": 0.2413793103448276, "education_science": 0.17647058823529413, "teacher_qualification": 0.30612244897959184, "high_school_politics": 0.4166666666666667, "high_school_geography": 0.041666666666666664, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.25, "logic": 0.14814814814814814, "law": 0.20689655172413793, "chinese_language_and_literature": 0.17857142857142858, "art_studies": 0.2631578947368421, "professional_tour_guide": 0.38235294117647056, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.25, "high_school_history": 0.32, "middle_school_history": 0.4074074074074074, "civil_servant": 0.23076923076923078, "sports_science": 0.2916666666666667, "plant_protection": 0.3333333333333333, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.3333333333333333, "fire_engineer": 0.1111111111111111, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.2777777777777778, "physician": 0.25925925925925924 } }, "prompt_5": { "accuracy": 0.24408468244084683, "category_acc": { "computer_network": 0.25, "operating_system": 0.2916666666666667, "computer_architecture": 0.19230769230769232, "college_programming": 0.30952380952380953, "college_physics": 0.25, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.27586206896551724, "high_school_mathematics": 0.08695652173913043, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.125, "middle_school_biology": 0.19230769230769232, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.2833333333333333, "business_administration": 0.18421052631578946, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.17647058823529413, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.375, "high_school_geography": 0.125, "middle_school_politics": 0.07692307692307693, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.125, "logic": 0.14814814814814814, "law": 0.27586206896551724, "chinese_language_and_literature": 0.14285714285714285, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.20588235294117646, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.25, "high_school_history": 0.2, "middle_school_history": 0.2222222222222222, "civil_servant": 0.25, "sports_science": 0.375, "plant_protection": 0.2962962962962963, "basic_medicine": 0.16666666666666666, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.19607843137254902, "accountant": 0.24074074074074073, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.24074074074074073, "physician": 0.25925925925925924 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.26881720430107525 }, "prompt_2": { "accuracy": 0.22939068100358423 }, "prompt_3": { "accuracy": 0.17562724014336917 }, "prompt_4": { "accuracy": 0.24014336917562723 }, "prompt_5": { "accuracy": 0.26881720430107525 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.25375582800897944, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.22297297297297297, "ancient_chinese": 0.2865853658536585, "arts": 0.275, "astronomy": 0.23030303030303031, "business_ethics": 0.23923444976076555, "chinese_civil_service_exam": 0.25, "chinese_driving_rule": 0.2595419847328244, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.27102803738317754, "chinese_history": 0.2724458204334365, "chinese_literature": 0.2549019607843137, "chinese_teacher_qualification": 0.24581005586592178, "clinical_knowledge": 0.2616033755274262, "college_actuarial_science": 0.2641509433962264, "college_education": 0.22429906542056074, "college_engineering_hydrology": 0.2169811320754717, "college_law": 0.24074074074074073, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2967032967032967, "computer_science": 0.2647058823529412, "computer_security": 0.2573099415204678, "conceptual_physics": 0.2585034013605442, "construction_project_management": 0.2302158273381295, "economics": 0.22012578616352202, "education": 0.2085889570552147, "electrical_engineering": 0.2616279069767442, "elementary_chinese": 0.27380952380952384, "elementary_commonsense": 0.2676767676767677, "elementary_information_and_technology": 0.2689075630252101, "elementary_mathematics": 0.26956521739130435, "ethnology": 0.3037037037037037, "food_science": 0.23076923076923078, "genetics": 0.25, "global_facts": 0.24161073825503357, "high_school_biology": 0.22485207100591717, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.2457627118644068, "high_school_mathematics": 0.17073170731707318, "high_school_physics": 0.21818181818181817, "high_school_politics": 0.2097902097902098, "human_sexuality": 0.3333333333333333, "international_law": 0.22162162162162163, "journalism": 0.27906976744186046, "jurisprudence": 0.2749391727493917, "legal_and_moral_basis": 0.2850467289719626, "logical": 0.21951219512195122, "machine_learning": 0.1721311475409836, "management": 0.24761904761904763, "marketing": 0.2611111111111111, "marxist_theory": 0.2698412698412698, "modern_chinese": 0.19827586206896552, "nutrition": 0.25517241379310346, "philosophy": 0.22857142857142856, "professional_accounting": 0.21714285714285714, "professional_law": 0.2985781990521327, "professional_medicine": 0.2898936170212766, "professional_psychology": 0.3146551724137931, "public_relations": 0.28160919540229884, "security_study": 0.2518518518518518, "sociology": 0.23008849557522124, "sports_science": 0.23030303030303031, "traditional_chinese_medicine": 0.1837837837837838, "virology": 0.23076923076923078, "world_history": 0.32298136645962733, "world_religions": 0.2625 } }, "prompt_2": { "accuracy": 0.2502158521844241, "category_acc": { "agronomy": 0.21893491124260356, "anatomy": 0.28378378378378377, "ancient_chinese": 0.27439024390243905, "arts": 0.275, "astronomy": 0.26666666666666666, "business_ethics": 0.2631578947368421, "chinese_civil_service_exam": 0.25, "chinese_driving_rule": 0.22137404580152673, "chinese_food_culture": 0.23529411764705882, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.25077399380804954, "chinese_literature": 0.25980392156862747, "chinese_teacher_qualification": 0.2681564245810056, "clinical_knowledge": 0.24050632911392406, "college_actuarial_science": 0.18867924528301888, "college_education": 0.2336448598130841, "college_engineering_hydrology": 0.25471698113207547, "college_law": 0.23148148148148148, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.25471698113207547, "college_medicine": 0.27106227106227104, "computer_science": 0.23529411764705882, "computer_security": 0.24561403508771928, "conceptual_physics": 0.19727891156462585, "construction_project_management": 0.2949640287769784, "economics": 0.25157232704402516, "education": 0.26380368098159507, "electrical_engineering": 0.20348837209302326, "elementary_chinese": 0.27380952380952384, "elementary_commonsense": 0.23232323232323232, "elementary_information_and_technology": 0.25630252100840334, "elementary_mathematics": 0.23478260869565218, "ethnology": 0.24444444444444444, "food_science": 0.24475524475524477, "genetics": 0.24431818181818182, "global_facts": 0.24161073825503357, "high_school_biology": 0.24260355029585798, "high_school_chemistry": 0.32575757575757575, "high_school_geography": 0.22033898305084745, "high_school_mathematics": 0.27439024390243905, "high_school_physics": 0.21818181818181817, "high_school_politics": 0.24475524475524477, "human_sexuality": 0.23809523809523808, "international_law": 0.22162162162162163, "journalism": 0.26744186046511625, "jurisprudence": 0.25304136253041365, "legal_and_moral_basis": 0.21962616822429906, "logical": 0.2682926829268293, "machine_learning": 0.2459016393442623, "management": 0.22380952380952382, "marketing": 0.26666666666666666, "marxist_theory": 0.2698412698412698, "modern_chinese": 0.20689655172413793, "nutrition": 0.18620689655172415, "philosophy": 0.3142857142857143, "professional_accounting": 0.29714285714285715, "professional_law": 0.2796208530805687, "professional_medicine": 0.26861702127659576, "professional_psychology": 0.21551724137931033, "public_relations": 0.2471264367816092, "security_study": 0.31851851851851853, "sociology": 0.23893805309734514, "sports_science": 0.26666666666666666, "traditional_chinese_medicine": 0.23243243243243245, "virology": 0.26627218934911245, "world_history": 0.2484472049689441, "world_religions": 0.25625 } }, "prompt_3": { "accuracy": 0.24650319461232947, "category_acc": { "agronomy": 0.20710059171597633, "anatomy": 0.2702702702702703, "ancient_chinese": 0.2682926829268293, "arts": 0.2375, "astronomy": 0.24242424242424243, "business_ethics": 0.18660287081339713, "chinese_civil_service_exam": 0.2625, "chinese_driving_rule": 0.22900763358778625, "chinese_food_culture": 0.22794117647058823, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.25077399380804954, "chinese_literature": 0.2549019607843137, "chinese_teacher_qualification": 0.24581005586592178, "clinical_knowledge": 0.2320675105485232, "college_actuarial_science": 0.29245283018867924, "college_education": 0.21495327102803738, "college_engineering_hydrology": 0.1509433962264151, "college_law": 0.16666666666666666, "college_mathematics": 0.19047619047619047, "college_medical_statistics": 0.25471698113207547, "college_medicine": 0.27106227106227104, "computer_science": 0.20588235294117646, "computer_security": 0.24561403508771928, "conceptual_physics": 0.23809523809523808, "construction_project_management": 0.2446043165467626, "economics": 0.29559748427672955, "education": 0.2085889570552147, "electrical_engineering": 0.2616279069767442, "elementary_chinese": 0.29365079365079366, "elementary_commonsense": 0.21212121212121213, "elementary_information_and_technology": 0.25630252100840334, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.26666666666666666, "food_science": 0.23776223776223776, "genetics": 0.2727272727272727, "global_facts": 0.26174496644295303, "high_school_biology": 0.2485207100591716, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.2818181818181818, "high_school_politics": 0.22377622377622378, "human_sexuality": 0.24603174603174602, "international_law": 0.23783783783783785, "journalism": 0.3081395348837209, "jurisprudence": 0.27007299270072993, "legal_and_moral_basis": 0.2803738317757009, "logical": 0.21951219512195122, "machine_learning": 0.1885245901639344, "management": 0.23333333333333334, "marketing": 0.23333333333333334, "marxist_theory": 0.26455026455026454, "modern_chinese": 0.1810344827586207, "nutrition": 0.22758620689655173, "philosophy": 0.22857142857142856, "professional_accounting": 0.2571428571428571, "professional_law": 0.25118483412322273, "professional_medicine": 0.2765957446808511, "professional_psychology": 0.27155172413793105, "public_relations": 0.2988505747126437, "security_study": 0.2222222222222222, "sociology": 0.26991150442477874, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.2594594594594595, "virology": 0.26627218934911245, "world_history": 0.2111801242236025, "world_religions": 0.21875 } }, "prompt_4": { "accuracy": 0.24624417199102056, "category_acc": { "agronomy": 0.3136094674556213, "anatomy": 0.22972972972972974, "ancient_chinese": 0.18292682926829268, "arts": 0.275, "astronomy": 0.17575757575757575, "business_ethics": 0.18181818181818182, "chinese_civil_service_exam": 0.275, "chinese_driving_rule": 0.33587786259541985, "chinese_food_culture": 0.25735294117647056, "chinese_foreign_policy": 0.16822429906542055, "chinese_history": 0.23529411764705882, "chinese_literature": 0.27450980392156865, "chinese_teacher_qualification": 0.2849162011173184, "clinical_knowledge": 0.2869198312236287, "college_actuarial_science": 0.18867924528301888, "college_education": 0.2336448598130841, "college_engineering_hydrology": 0.24528301886792453, "college_law": 0.25925925925925924, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.24528301886792453, "college_medicine": 0.23809523809523808, "computer_science": 0.25980392156862747, "computer_security": 0.23976608187134502, "conceptual_physics": 0.22448979591836735, "construction_project_management": 0.2949640287769784, "economics": 0.27044025157232704, "education": 0.2331288343558282, "electrical_engineering": 0.26744186046511625, "elementary_chinese": 0.2222222222222222, "elementary_commonsense": 0.30808080808080807, "elementary_information_and_technology": 0.24789915966386555, "elementary_mathematics": 0.24782608695652175, "ethnology": 0.3037037037037037, "food_science": 0.2867132867132867, "genetics": 0.23295454545454544, "global_facts": 0.20134228187919462, "high_school_biology": 0.26627218934911245, "high_school_chemistry": 0.22727272727272727, "high_school_geography": 0.22033898305084745, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.1888111888111888, "human_sexuality": 0.24603174603174602, "international_law": 0.22702702702702704, "journalism": 0.28488372093023256, "jurisprudence": 0.22384428223844283, "legal_and_moral_basis": 0.22897196261682243, "logical": 0.3170731707317073, "machine_learning": 0.29508196721311475, "management": 0.2619047619047619, "marketing": 0.2722222222222222, "marxist_theory": 0.2222222222222222, "modern_chinese": 0.21551724137931033, "nutrition": 0.1793103448275862, "philosophy": 0.18095238095238095, "professional_accounting": 0.24, "professional_law": 0.23222748815165878, "professional_medicine": 0.24202127659574468, "professional_psychology": 0.27586206896551724, "public_relations": 0.28735632183908044, "security_study": 0.2222222222222222, "sociology": 0.19469026548672566, "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.2702702702702703, "virology": 0.26627218934911245, "world_history": 0.2608695652173913, "world_religions": 0.225 } }, "prompt_5": { "accuracy": 0.24641685373855984, "category_acc": { "agronomy": 0.21893491124260356, "anatomy": 0.25675675675675674, "ancient_chinese": 0.2804878048780488, "arts": 0.3, "astronomy": 0.2545454545454545, "business_ethics": 0.22009569377990432, "chinese_civil_service_exam": 0.225, "chinese_driving_rule": 0.16793893129770993, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.21495327102803738, "chinese_history": 0.23839009287925697, "chinese_literature": 0.22549019607843138, "chinese_teacher_qualification": 0.29608938547486036, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.2169811320754717, "college_education": 0.2523364485981308, "college_engineering_hydrology": 0.2358490566037736, "college_law": 0.23148148148148148, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.20754716981132076, "college_medicine": 0.23076923076923078, "computer_science": 0.27941176470588236, "computer_security": 0.24561403508771928, "conceptual_physics": 0.2789115646258503, "construction_project_management": 0.18705035971223022, "economics": 0.25157232704402516, "education": 0.1901840490797546, "electrical_engineering": 0.20930232558139536, "elementary_chinese": 0.23015873015873015, "elementary_commonsense": 0.23737373737373738, "elementary_information_and_technology": 0.29411764705882354, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.2740740740740741, "food_science": 0.25874125874125875, "genetics": 0.22727272727272727, "global_facts": 0.3087248322147651, "high_school_biology": 0.21893491124260356, "high_school_chemistry": 0.18181818181818182, "high_school_geography": 0.2542372881355932, "high_school_mathematics": 0.3048780487804878, "high_school_physics": 0.2, "high_school_politics": 0.2727272727272727, "human_sexuality": 0.2222222222222222, "international_law": 0.24324324324324326, "journalism": 0.22674418604651161, "jurisprudence": 0.19951338199513383, "legal_and_moral_basis": 0.308411214953271, "logical": 0.2682926829268293, "machine_learning": 0.3114754098360656, "management": 0.23333333333333334, "marketing": 0.28888888888888886, "marxist_theory": 0.2698412698412698, "modern_chinese": 0.21551724137931033, "nutrition": 0.2827586206896552, "philosophy": 0.26666666666666666, "professional_accounting": 0.21142857142857144, "professional_law": 0.2796208530805687, "professional_medicine": 0.26861702127659576, "professional_psychology": 0.22844827586206898, "public_relations": 0.21839080459770116, "security_study": 0.26666666666666666, "sociology": 0.22123893805309736, "sports_science": 0.2606060606060606, "traditional_chinese_medicine": 0.23243243243243245, "virology": 0.2603550295857988, "world_history": 0.2732919254658385, "world_religions": 0.2625 } } }, "zbench": { "prompt_1": { "accuracy": 0.21212121212121213 }, "prompt_2": { "accuracy": 0.21212121212121213 }, "prompt_3": { "accuracy": 0.21212121212121213 }, "prompt_4": { "accuracy": 0.12121212121212122 }, "prompt_5": { "accuracy": 0.15151515151515152 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.1318181818181818 }, "prompt_2": { "accuracy": 0.16136363636363638 }, "prompt_3": { "accuracy": 0.16136363636363638 }, "prompt_4": { "accuracy": 0.17727272727272728 }, "prompt_5": { "accuracy": 0.17272727272727273 } }, "ocnli": { "prompt_1": { "accuracy": 0.34 }, "prompt_2": { "accuracy": 0.3254237288135593 }, "prompt_3": { "accuracy": 0.34067796610169493 }, "prompt_4": { "accuracy": 0.34271186440677964 }, "prompt_5": { "accuracy": 0.33559322033898303 } }, "c3": { "prompt_1": { "accuracy": 0.2774869109947644 }, "prompt_2": { "accuracy": 0.26925953627524307 }, "prompt_3": { "accuracy": 0.27299925205684367 }, "prompt_4": { "accuracy": 0.27486910994764396 }, "prompt_5": { "accuracy": 0.28421839940164545 } }, "dream": { "prompt_1": { "accuracy": 0.34296913277804997 }, "prompt_2": { "accuracy": 0.3522782949534542 }, "prompt_3": { "accuracy": 0.3297403233708966 }, "prompt_4": { "accuracy": 0.3400293973542381 }, "prompt_5": { "accuracy": 0.3395394414502695 } }, "samsum": { "prompt_1": { "rouge1": 0.08732788348986167, "rouge2": 0.0014923956624199408, "rougeL": 0.07139937267026018, "avg_rouge": 0.05340655060751393 }, "prompt_2": { "rouge1": 0.09429276780188824, "rouge2": 0.0015571897217856114, "rougeL": 0.07590220186880957, "avg_rouge": 0.05725071979749447 }, "prompt_3": { "rouge1": 0.09149543908197538, "rouge2": 0.0013711964459392765, "rougeL": 0.07469201145443315, "avg_rouge": 0.05585288232744926 }, "prompt_4": { "rouge1": 0.08950232953091344, "rouge2": 0.00146326662774582, "rougeL": 0.07261221266441964, "avg_rouge": 0.054525936274359636 }, "prompt_5": { "rouge1": 0.09373426610282494, "rouge2": 0.0016362694068140476, "rougeL": 0.07685534271487146, "avg_rouge": 0.05740862607483682 } }, "dialogsum": { "prompt_1": { "rouge1": 0.10636719666265786, "rouge2": 0.001910313699982673, "rougeL": 0.08618104814226087, "avg_rouge": 0.0648195195016338 }, "prompt_2": { "rouge1": 0.1075911884138222, "rouge2": 0.002064888511841431, "rougeL": 0.08842720122501456, "avg_rouge": 0.0660277593835594 }, "prompt_3": { "rouge1": 0.10375648507527026, "rouge2": 0.0016752661752168182, "rougeL": 0.08609410346879956, "avg_rouge": 0.06384195157309554 }, "prompt_4": { "rouge1": 0.1044636520106956, "rouge2": 0.0016187605912823585, "rougeL": 0.08534402243673829, "avg_rouge": 0.06380881167957209 }, "prompt_5": { "rouge1": 0.10376988698752433, "rouge2": 0.0012622705102603058, "rougeL": 0.08537471802223869, "avg_rouge": 0.06346895850667444 } }, "sst2": { "prompt_1": { "accuracy": 0.518348623853211 }, "prompt_2": { "accuracy": 0.4690366972477064 }, "prompt_3": { "accuracy": 0.481651376146789 }, "prompt_4": { "accuracy": 0.49311926605504586 }, "prompt_5": { "accuracy": 0.5206422018348624 } }, "cola": { "prompt_1": { "accuracy": 0.49089165867689355 }, "prompt_2": { "accuracy": 0.4937679769894535 }, "prompt_3": { "accuracy": 0.513902205177373 }, "prompt_4": { "accuracy": 0.5244487056567594 }, "prompt_5": { "accuracy": 0.49952061361457334 } }, "qqp": { "prompt_1": { "accuracy": 0.498 }, "prompt_2": { "accuracy": 0.5065 }, "prompt_3": { "accuracy": 0.4915 }, "prompt_4": { "accuracy": 0.508 }, "prompt_5": { "accuracy": 0.4905 } }, "mnli": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.5 } }, "qnli": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.5 } }, "wnli": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.4 }, "prompt_5": { "accuracy": 0.6 } }, "rte": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.5 } }, "mrpc": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.2573529411764706, "language_acc": { "Spanish": 0.2596638655462185, "English": 0.2596638655462185, "Chinese": 0.2453781512605042, "Vietnamese": 0.2647058823529412 }, "consistency_score_2": 0.24971988795518207, "consistency_score_3": 0.06302521008403361, "consistency_score_4": 0.015966386554621848, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.25882352941176473, "Spanish,Chinese": 0.2605042016806723, "Spanish,Vietnamese": 0.25546218487394956, "English,Chinese": 0.27310924369747897, "English,Vietnamese": 0.20588235294117646, "Chinese,Vietnamese": 0.2445378151260504 }, "3_combine": { "Spanish,English,Chinese": 0.06890756302521009, "Spanish,English,Vietnamese": 0.05546218487394958, "Spanish,Chinese,Vietnamese": 0.073109243697479, "English,Chinese,Vietnamese": 0.0546218487394958 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.015966386554621848 } }, "AC3_2": 0.25347896369405065, "AC3_3": 0.10125361616897419, "AC3_4": 0.03006736898117714 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.2552380952380952, "language_acc": { "Filipino": 0.23333333333333334, "Vietnamese": 0.26666666666666666, "Chinese": 0.24666666666666667, "Spanish": 0.26666666666666666, "Malay": 0.32666666666666666, "Indonesian": 0.2, "English": 0.24666666666666667 }, "consistency_score_2": 0.24603174603174605, "consistency_score_3": 0.05809523809523809, "consistency_score_4": 0.011999999999999995, "consistency_score_5": 0.0019047619047619048, "consistency_score_6": 0.0, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.25333333333333335, "Filipino,Chinese": 0.17333333333333334, "Filipino,Spanish": 0.24, "Filipino,Malay": 0.26, "Filipino,Indonesian": 0.28, "Filipino,English": 0.2733333333333333, "Vietnamese,Chinese": 0.2733333333333333, "Vietnamese,Spanish": 0.2733333333333333, "Vietnamese,Malay": 0.28, "Vietnamese,Indonesian": 0.23333333333333334, "Vietnamese,English": 0.24666666666666667, "Chinese,Spanish": 0.29333333333333333, "Chinese,Malay": 0.28, "Chinese,Indonesian": 0.24666666666666667, "Chinese,English": 0.26666666666666666, "Spanish,Malay": 0.20666666666666667, "Spanish,Indonesian": 0.19333333333333333, "Spanish,English": 0.22666666666666666, "Malay,Indonesian": 0.20666666666666667, "Malay,English": 0.21333333333333335, "Indonesian,English": 0.24666666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.04666666666666667, "Filipino,Vietnamese,Spanish": 0.08, "Filipino,Vietnamese,Malay": 0.08, "Filipino,Vietnamese,Indonesian": 0.06666666666666667, "Filipino,Vietnamese,English": 0.06, "Filipino,Chinese,Spanish": 0.04666666666666667, "Filipino,Chinese,Malay": 0.04, "Filipino,Chinese,Indonesian": 0.06666666666666667, "Filipino,Chinese,English": 0.04666666666666667, "Filipino,Spanish,Malay": 0.04, "Filipino,Spanish,Indonesian": 0.05333333333333334, "Filipino,Spanish,English": 0.05333333333333334, "Filipino,Malay,Indonesian": 0.05333333333333334, "Filipino,Malay,English": 0.06, "Filipino,Indonesian,English": 0.07333333333333333, "Vietnamese,Chinese,Spanish": 0.08, "Vietnamese,Chinese,Malay": 0.10666666666666667, "Vietnamese,Chinese,Indonesian": 0.09333333333333334, "Vietnamese,Chinese,English": 0.06, "Vietnamese,Spanish,Malay": 0.04666666666666667, "Vietnamese,Spanish,Indonesian": 0.06666666666666667, "Vietnamese,Spanish,English": 0.04666666666666667, "Vietnamese,Malay,Indonesian": 0.05333333333333334, "Vietnamese,Malay,English": 0.04666666666666667, "Vietnamese,Indonesian,English": 0.06, "Chinese,Spanish,Malay": 0.06, "Chinese,Spanish,Indonesian": 0.06666666666666667, "Chinese,Spanish,English": 0.06666666666666667, "Chinese,Malay,Indonesian": 0.03333333333333333, "Chinese,Malay,English": 0.04666666666666667, "Chinese,Indonesian,English": 0.06, "Spanish,Malay,Indonesian": 0.04666666666666667, "Spanish,Malay,English": 0.02, "Spanish,Indonesian,English": 0.04666666666666667, "Malay,Indonesian,English": 0.06 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.02, "Filipino,Vietnamese,Chinese,Malay": 0.02, "Filipino,Vietnamese,Chinese,Indonesian": 0.02666666666666667, "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.02, "Filipino,Vietnamese,Spanish,Indonesian": 0.03333333333333333, "Filipino,Vietnamese,Spanish,English": 0.006666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.02, "Filipino,Vietnamese,Malay,English": 0.006666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, "Filipino,Chinese,Spanish,Malay": 0.0, "Filipino,Chinese,Spanish,Indonesian": 0.02666666666666667, "Filipino,Chinese,Spanish,English": 0.0, "Filipino,Chinese,Malay,Indonesian": 0.006666666666666667, "Filipino,Chinese,Malay,English": 0.0, "Filipino,Chinese,Indonesian,English": 0.006666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.0, "Filipino,Spanish,Malay,English": 0.006666666666666667, "Filipino,Spanish,Indonesian,English": 0.006666666666666667, "Filipino,Malay,Indonesian,English": 0.013333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.013333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.03333333333333333, "Vietnamese,Chinese,Spanish,English": 0.006666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.02, "Vietnamese,Chinese,Malay,English": 0.006666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.02666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.013333333333333334, "Vietnamese,Spanish,Malay,English": 0.006666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.02, "Vietnamese,Malay,Indonesian,English": 0.013333333333333334, "Chinese,Spanish,Malay,Indonesian": 0.0, "Chinese,Spanish,Malay,English": 0.0, "Chinese,Spanish,Indonesian,English": 0.006666666666666667, "Chinese,Malay,Indonesian,English": 0.006666666666666667, "Spanish,Malay,Indonesian,English": 0.013333333333333334 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.02, "Filipino,Vietnamese,Chinese,Spanish,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Chinese,Spanish,Malay,English": 0.0, "Filipino,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.006666666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Vietnamese,Spanish,Malay,Indonesian,English": 0.006666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 } }, "AC3_2": 0.25055037842946437, "AC3_3": 0.09464756112191373, "AC3_4": 0.022922309328557215, "AC3_5": 0.0037813051131679403, "AC3_6": 0.0, "AC3_7": 0.0 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.24756493506493502, "language_acc": { "Indonesian": 0.2897727272727273, "English": 0.2784090909090909, "Filipino": 0.25, "Spanish": 0.2784090909090909, "Chinese": 0.2556818181818182, "Malay": 0.19886363636363635, "Vietnamese": 0.18181818181818182 }, "consistency_score_2": 0.25216450216450215, "consistency_score_3": 0.06428571428571428, "consistency_score_4": 0.01655844155844155, "consistency_score_5": 0.004058441558441559, "consistency_score_6": 0.0008116883116883117, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.25, "Indonesian,Filipino": 0.23863636363636365, "Indonesian,Spanish": 0.26136363636363635, "Indonesian,Chinese": 0.23863636363636365, "Indonesian,Malay": 0.26136363636363635, "Indonesian,Vietnamese": 0.2727272727272727, "English,Filipino": 0.2215909090909091, "English,Spanish": 0.2556818181818182, "English,Chinese": 0.23295454545454544, "English,Malay": 0.23295454545454544, "English,Vietnamese": 0.2215909090909091, "Filipino,Spanish": 0.2727272727272727, "Filipino,Chinese": 0.26136363636363635, "Filipino,Malay": 0.26136363636363635, "Filipino,Vietnamese": 0.2556818181818182, "Spanish,Chinese": 0.2556818181818182, "Spanish,Malay": 0.30113636363636365, "Spanish,Vietnamese": 0.19886363636363635, "Chinese,Malay": 0.2727272727272727, "Chinese,Vietnamese": 0.26136363636363635, "Malay,Vietnamese": 0.26704545454545453 }, "3_combine": { "Indonesian,English,Filipino": 0.05113636363636364, "Indonesian,English,Spanish": 0.056818181818181816, "Indonesian,English,Chinese": 0.0625, "Indonesian,English,Malay": 0.056818181818181816, "Indonesian,English,Vietnamese": 0.05113636363636364, "Indonesian,Filipino,Spanish": 0.0625, "Indonesian,Filipino,Chinese": 0.07386363636363637, "Indonesian,Filipino,Malay": 0.0625, "Indonesian,Filipino,Vietnamese": 0.0625, "Indonesian,Spanish,Chinese": 0.0625, "Indonesian,Spanish,Malay": 0.08522727272727272, "Indonesian,Spanish,Vietnamese": 0.056818181818181816, "Indonesian,Chinese,Malay": 0.07386363636363637, "Indonesian,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Spanish": 0.06818181818181818, "English,Filipino,Chinese": 0.05113636363636364, "English,Filipino,Malay": 0.05113636363636364, "English,Filipino,Vietnamese": 0.03977272727272727, "English,Spanish,Chinese": 0.0625, "English,Spanish,Malay": 0.07386363636363637, "English,Spanish,Vietnamese": 0.045454545454545456, "English,Chinese,Malay": 0.05113636363636364, "English,Chinese,Vietnamese": 0.028409090909090908, "English,Malay,Vietnamese": 0.03977272727272727, "Filipino,Spanish,Chinese": 0.06818181818181818, "Filipino,Spanish,Malay": 0.06818181818181818, "Filipino,Spanish,Vietnamese": 0.0625, "Filipino,Chinese,Malay": 0.07954545454545454, "Filipino,Chinese,Vietnamese": 0.08522727272727272, "Filipino,Malay,Vietnamese": 0.07954545454545454, "Spanish,Chinese,Malay": 0.10795454545454546, "Spanish,Chinese,Vietnamese": 0.056818181818181816, "Spanish,Malay,Vietnamese": 0.07954545454545454, "Chinese,Malay,Vietnamese": 0.07954545454545454 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.022727272727272728, "Indonesian,English,Filipino,Chinese": 0.017045454545454544, "Indonesian,English,Filipino,Malay": 0.005681818181818182, "Indonesian,English,Filipino,Vietnamese": 0.011363636363636364, "Indonesian,English,Spanish,Chinese": 0.011363636363636364, "Indonesian,English,Spanish,Malay": 0.017045454545454544, "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, "Indonesian,English,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Malay": 0.017045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Chinese,Malay": 0.028409090909090908, "Indonesian,Filipino,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Spanish,Chinese,Malay": 0.028409090909090908, "Indonesian,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,Spanish,Malay,Vietnamese": 0.028409090909090908, "Indonesian,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese": 0.017045454545454544, "English,Filipino,Spanish,Malay": 0.005681818181818182, "English,Filipino,Spanish,Vietnamese": 0.017045454545454544, "English,Filipino,Chinese,Malay": 0.005681818181818182, "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, "English,Filipino,Malay,Vietnamese": 0.011363636363636364, "English,Spanish,Chinese,Malay": 0.022727272727272728, "English,Spanish,Chinese,Vietnamese": 0.005681818181818182, "English,Spanish,Malay,Vietnamese": 0.017045454545454544, "English,Chinese,Malay,Vietnamese": 0.005681818181818182, "Filipino,Spanish,Chinese,Malay": 0.022727272727272728, "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, "Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Malay": 0.0, "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.0, "English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, "AC3_2": 0.24984355109295978, "AC3_3": 0.1020673755900859, "AC3_4": 0.031040717107637145, "AC3_5": 0.007985965644082193, "AC3_6": 0.0016180714703611886, "AC3_7": 0.0 } }, "sg_eval": { "prompt_1": { "accuracy": 0.2524271844660194 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2857142857142857 } }, "us_eval": { "prompt_1": { "accuracy": 0.2523364485981308 } }, "ph_eval": { "prompt_1": { "accuracy": 0.21, "category_acc": { "brand": 0.4, "demographics": 0.4, "biology": 0.2, "history": 0.13333333333333333, "literature": 0.5, "politics": 0.1, "culture": 0.1, "film": 0.0, "law": 0.2, "geography": 0.2 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.008866968704698027 } }, "indommlu": { "prompt_1": { "accuracy": 0.23179117431070165, "category_acc": { "History": 0.1566265060240964, "Geography": 0.18571428571428572, "Lampungic": 0.2925170068027211, "Social science": 0.24707846410684475, "Balinese": 0.267515923566879, "Makassarese": 0.23118279569892472, "Banjarese": 0.2986111111111111, "Chemistry": 0.2029197080291971, "Biology": 0.18106508875739644, "Science": 0.23426212590299278, "Christian religion": 0.22885572139303484, "Art": 0.24459234608985025, "Islam religion": 0.24466571834992887, "Hindu religion": 0.24666666666666667, "Madurese": 0.21694915254237288, "Sport": 0.2635135135135135, "Indonesian language": 0.2425280199252802, "Physics": 0.2383838383838384, "Minangkabau culture": 0.23115577889447236, "Dayak language": 0.21100917431192662, "Sociology": 0.18548387096774194, "Economy": 0.18442622950819673, "Sundanese": 0.27398444252376836, "Javanese": 0.25201612903225806, "Civic education": 0.23032904148783978 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.009307104377338064 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.009288422226243908 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.009267355508947273 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.009239626132587908 } }, "mmlu": { "prompt_1": { "accuracy": 0.2660443407234539 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.24640686449767607, "category_acc": { "high_school_european_history": 0.25609756097560976, "business_ethics": 0.2222222222222222, "clinical_knowledge": 0.3143939393939394, "medical_genetics": 0.2222222222222222, "high_school_us_history": 0.24630541871921183, "high_school_physics": 0.30666666666666664, "high_school_world_history": 0.2542372881355932, "virology": 0.2606060606060606, "high_school_microeconomics": 0.1940928270042194, "econometrics": 0.23893805309734514, "college_computer_science": 0.26262626262626265, "high_school_biology": 0.24919093851132687, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.24555160142348753, "philosophy": 0.25161290322580643, "professional_medicine": 0.23247232472324722, "nutrition": 0.24262295081967214, "global_facts": 0.21212121212121213, "machine_learning": 0.17117117117117117, "security_studies": 0.2581967213114754, "public_relations": 0.24770642201834864, "professional_psychology": 0.23076923076923078, "prehistory": 0.25077399380804954, "anatomy": 0.29850746268656714, "human_sexuality": 0.25384615384615383, "college_medicine": 0.28488372093023256, "high_school_government_and_politics": 0.21875, "college_chemistry": 0.21212121212121213, "logical_fallacies": 0.2654320987654321, "high_school_geography": 0.28426395939086296, "elementary_mathematics": 0.246684350132626, "human_aging": 0.22072072072072071, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.27205882352941174, "formal_logic": 0.32, "high_school_statistics": 0.2372093023255814, "international_law": 0.225, "high_school_mathematics": 0.241635687732342, "high_school_computer_science": 0.20202020202020202, "conceptual_physics": 0.27350427350427353, "miscellaneous": 0.23785166240409208, "high_school_chemistry": 0.2524752475247525, "marketing": 0.26609442060085836, "professional_law": 0.23613829093281147, "management": 0.2647058823529412, "college_physics": 0.16831683168316833, "jurisprudence": 0.19626168224299065, "world_religions": 0.23529411764705882, "sociology": 0.23, "us_foreign_policy": 0.25252525252525254, "high_school_macroeconomics": 0.23136246786632392, "computer_security": 0.2727272727272727, "moral_scenarios": 0.24049217002237136, "moral_disputes": 0.24057971014492754, "electrical_engineering": 0.2916666666666667, "astronomy": 0.2185430463576159, "college_biology": 0.2727272727272727 } } }, "c_eval": { "prompt_1": { "accuracy": 0.24294205052005943 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.2752179327521793, "category_acc": { "computer_network": 0.375, "operating_system": 0.20833333333333334, "computer_architecture": 0.19230769230769232, "college_programming": 0.23809523809523808, "college_physics": 0.2916666666666667, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.23076923076923078, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.4642857142857143, "college_economics": 0.35, "business_administration": 0.3157894736842105, "marxism": 0.25, "mao_zedong_thought": 0.2413793103448276, "education_science": 0.3235294117647059, "teacher_qualification": 0.42857142857142855, "high_school_politics": 0.20833333333333334, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.17857142857142858, "ideological_and_moral_cultivation": 0.125, "logic": 0.2962962962962963, "law": 0.3448275862068966, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.3157894736842105, "professional_tour_guide": 0.3235294117647059, "legal_professional": 0.17857142857142858, "high_school_chinese": 0.08333333333333333, "high_school_history": 0.32, "middle_school_history": 0.2962962962962963, "civil_servant": 0.2692307692307692, "sports_science": 0.3333333333333333, "plant_protection": 0.25925925925925924, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.2962962962962963, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.16666666666666666, "tax_accountant": 0.3148148148148148, "physician": 0.3148148148148148 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.27956989247311825 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.24987048868934553, "category_acc": { "agronomy": 0.26627218934911245, "anatomy": 0.22972972972972974, "ancient_chinese": 0.21951219512195122, "arts": 0.225, "astronomy": 0.26666666666666666, "business_ethics": 0.2727272727272727, "chinese_civil_service_exam": 0.2875, "chinese_driving_rule": 0.2366412213740458, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.308411214953271, "chinese_history": 0.2260061919504644, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.22346368715083798, "clinical_knowledge": 0.26582278481012656, "college_actuarial_science": 0.2169811320754717, "college_education": 0.2523364485981308, "college_engineering_hydrology": 0.27358490566037735, "college_law": 0.17592592592592593, "college_mathematics": 0.3142857142857143, "college_medical_statistics": 0.24528301886792453, "college_medicine": 0.20512820512820512, "computer_science": 0.23039215686274508, "computer_security": 0.1871345029239766, "conceptual_physics": 0.2585034013605442, "construction_project_management": 0.17985611510791366, "economics": 0.1761006289308176, "education": 0.22085889570552147, "electrical_engineering": 0.26744186046511625, "elementary_chinese": 0.25793650793650796, "elementary_commonsense": 0.20202020202020202, "elementary_information_and_technology": 0.3067226890756303, "elementary_mathematics": 0.2391304347826087, "ethnology": 0.21481481481481482, "food_science": 0.3006993006993007, "genetics": 0.24431818181818182, "global_facts": 0.2550335570469799, "high_school_biology": 0.2603550295857988, "high_school_chemistry": 0.23484848484848486, "high_school_geography": 0.2796610169491525, "high_school_mathematics": 0.2865853658536585, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.2727272727272727, "human_sexuality": 0.23809523809523808, "international_law": 0.2756756756756757, "journalism": 0.23255813953488372, "jurisprudence": 0.2773722627737226, "legal_and_moral_basis": 0.29439252336448596, "logical": 0.2764227642276423, "machine_learning": 0.2786885245901639, "management": 0.3238095238095238, "marketing": 0.2611111111111111, "marxist_theory": 0.2275132275132275, "modern_chinese": 0.3275862068965517, "nutrition": 0.2689655172413793, "philosophy": 0.2761904761904762, "professional_accounting": 0.19428571428571428, "professional_law": 0.26540284360189575, "professional_medicine": 0.2579787234042553, "professional_psychology": 0.23275862068965517, "public_relations": 0.20689655172413793, "security_study": 0.23703703703703705, "sociology": 0.2168141592920354, "sports_science": 0.2787878787878788, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.2603550295857988, "world_history": 0.2111801242236025, "world_religions": 0.225 } } }, "zbench": { "prompt_1": { "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.17272727272727273 } }, "ocnli": { "prompt_1": { "accuracy": 0.32440677966101694 } }, "c3": { "prompt_1": { "accuracy": 0.2782348541510845 } }, "dream": { "prompt_1": { "accuracy": 0.3390494855463008 } }, "samsum": { "prompt_1": { "rouge1": 0.0895541253352484, "rouge2": 0.0015213583740047467, "rougeL": 0.0719862724420868, "avg_rouge": 0.054353918717113314 } }, "dialogsum": { "prompt_1": { "rouge1": 0.10633392487487775, "rouge2": 0.0017154941667223626, "rougeL": 0.08807783273920188, "avg_rouge": 0.06537575059360066 } }, "sst2": { "prompt_1": { "accuracy": 0.49426605504587157 } }, "cola": { "prompt_1": { "accuracy": 0.50143815915628 } }, "qqp": { "prompt_1": { "accuracy": 0.506 } }, "mnli": { "prompt_1": { "accuracy": 0.2 } }, "qnli": { "prompt_1": { "accuracy": 0.5 } }, "wnli": { "prompt_1": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 0.2 } }, "mrpc": { "prompt_1": { "accuracy": 0.6 } } } }, "meta_llama_3_8b": { "model_size": "8B", "model_link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9142857142857143, "language_acc": { "Spanish": 0.9117647058823529, "English": 0.9302521008403362, "Chinese": 0.9058823529411765, "Vietnamese": 0.9092436974789916 }, "consistency_score_2": 0.9068627450980392, "consistency_score_3": 0.8649159663865547, "consistency_score_4": 0.8369747899159664, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9294117647058824, "Spanish,Chinese": 0.8907563025210085, "Spanish,Vietnamese": 0.9042016806722689, "English,Chinese": 0.9092436974789916, "English,Vietnamese": 0.9159663865546218, "Chinese,Vietnamese": 0.8915966386554622 }, "3_combine": { "Spanish,English,Chinese": 0.8680672268907563, "Spanish,English,Vietnamese": 0.880672268907563, "Spanish,Chinese,Vietnamese": 0.8487394957983193, "English,Chinese,Vietnamese": 0.8621848739495799 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8369747899159664 } }, "AC3_2": 0.9105591016957517, "AC3_3": 0.8889158780263842, "AC3_4": 0.8739237729141776 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.5561904761904761, "language_acc": { "Filipino": 0.5133333333333333, "Vietnamese": 0.5066666666666667, "Chinese": 0.5466666666666666, "Spanish": 0.5666666666666667, "Malay": 0.52, "Indonesian": 0.5733333333333334, "English": 0.6666666666666666 }, "consistency_score_2": 0.6019047619047619, "consistency_score_3": 0.4415238095238096, "consistency_score_4": 0.35180952380952385, "consistency_score_5": 0.29460317460317464, "consistency_score_6": 0.2571428571428572, "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5266666666666666, "Filipino,Chinese": 0.5533333333333333, "Filipino,Spanish": 0.6666666666666666, "Filipino,Malay": 0.5533333333333333, "Filipino,Indonesian": 0.6266666666666667, "Filipino,English": 0.6466666666666666, "Vietnamese,Chinese": 0.54, "Vietnamese,Spanish": 0.6, "Vietnamese,Malay": 0.6, "Vietnamese,Indonesian": 0.58, "Vietnamese,English": 0.6133333333333333, "Chinese,Spanish": 0.5733333333333334, "Chinese,Malay": 0.5533333333333333, "Chinese,Indonesian": 0.56, "Chinese,English": 0.58, "Spanish,Malay": 0.6, "Spanish,Indonesian": 0.64, "Spanish,English": 0.7133333333333334, "Malay,Indonesian": 0.6133333333333333, "Malay,English": 0.6466666666666666, "Indonesian,English": 0.6533333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.36666666666666664, "Filipino,Vietnamese,Spanish": 0.43333333333333335, "Filipino,Vietnamese,Malay": 0.4, "Filipino,Vietnamese,Indonesian": 0.4, "Filipino,Vietnamese,English": 0.43333333333333335, "Filipino,Chinese,Spanish": 0.43333333333333335, "Filipino,Chinese,Malay": 0.38, "Filipino,Chinese,Indonesian": 0.4, "Filipino,Chinese,English": 0.42, "Filipino,Spanish,Malay": 0.4666666666666667, "Filipino,Spanish,Indonesian": 0.5, "Filipino,Spanish,English": 0.5466666666666666, "Filipino,Malay,Indonesian": 0.4266666666666667, "Filipino,Malay,English": 0.4666666666666667, "Filipino,Indonesian,English": 0.48, "Vietnamese,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Chinese,Malay": 0.38666666666666666, "Vietnamese,Chinese,Indonesian": 0.4066666666666667, "Vietnamese,Chinese,English": 0.42, "Vietnamese,Spanish,Malay": 0.44666666666666666, "Vietnamese,Spanish,Indonesian": 0.44, "Vietnamese,Spanish,English": 0.4866666666666667, "Vietnamese,Malay,Indonesian": 0.44666666666666666, "Vietnamese,Malay,English": 0.47333333333333333, "Vietnamese,Indonesian,English": 0.4533333333333333, "Chinese,Spanish,Malay": 0.4, "Chinese,Spanish,Indonesian": 0.41333333333333333, "Chinese,Spanish,English": 0.4666666666666667, "Chinese,Malay,Indonesian": 0.41333333333333333, "Chinese,Malay,English": 0.42, "Chinese,Indonesian,English": 0.43333333333333335, "Spanish,Malay,Indonesian": 0.46, "Spanish,Malay,English": 0.5066666666666667, "Spanish,Indonesian,English": 0.5266666666666666, "Malay,Indonesian,English": 0.49333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Malay": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.3, "Filipino,Vietnamese,Chinese,English": 0.31333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.3466666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Spanish,English": 0.38666666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,Malay,English": 0.35333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.35333333333333333, "Filipino,Chinese,Spanish,Malay": 0.3333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.3466666666666667, "Filipino,Chinese,Spanish,English": 0.38, "Filipino,Chinese,Malay,Indonesian": 0.32, "Filipino,Chinese,Malay,English": 0.32, "Filipino,Chinese,Indonesian,English": 0.34, "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Spanish,Malay,English": 0.4066666666666667, "Filipino,Spanish,Indonesian,English": 0.43333333333333335, "Filipino,Malay,Indonesian,English": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.32, "Vietnamese,Chinese,Spanish,Indonesian": 0.32, "Vietnamese,Chinese,Spanish,English": 0.36, "Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, "Vietnamese,Chinese,Malay,English": 0.34, "Vietnamese,Chinese,Indonesian,English": 0.3466666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.38, "Vietnamese,Spanish,Malay,English": 0.38, "Vietnamese,Spanish,Indonesian,English": 0.3933333333333333, "Vietnamese,Malay,Indonesian,English": 0.38666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Chinese,Spanish,Malay,English": 0.34, "Chinese,Spanish,Indonesian,English": 0.36, "Chinese,Malay,Indonesian,English": 0.36, "Spanish,Malay,Indonesian,English": 0.4 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Chinese,Malay,English": 0.26, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Spanish,Malay,English": 0.31333333333333335, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.30666666666666664, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.31333333333333335, "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, "Vietnamese,Chinese,Spanish,Malay,English": 0.2866666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3, "Vietnamese,Chinese,Malay,Indonesian,English": 0.30666666666666664, "Vietnamese,Spanish,Malay,Indonesian,English": 0.34, "Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.24, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.26, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.29333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, "AC3_2": 0.5781453633585992, "AC3_3": 0.49226785936628614, "AC3_4": 0.4309980320581802, "AC3_5": 0.3851826580926087, "AC3_6": 0.35168952822708405, "AC3_7": 0.3287494973447703 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.49512987012987014, "language_acc": { "Indonesian": 0.4431818181818182, "English": 0.5625, "Filipino": 0.42613636363636365, "Spanish": 0.5170454545454546, "Chinese": 0.5227272727272727, "Malay": 0.4715909090909091, "Vietnamese": 0.5227272727272727 }, "consistency_score_2": 0.5687229437229437, "consistency_score_3": 0.39545454545454545, "consistency_score_4": 0.3003246753246754, "consistency_score_5": 0.24053030303030298, "consistency_score_6": 0.2012987012987013, "consistency_score_7": 0.17613636363636365, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5454545454545454, "Indonesian,Filipino": 0.5227272727272727, "Indonesian,Spanish": 0.5738636363636364, "Indonesian,Chinese": 0.5340909090909091, "Indonesian,Malay": 0.6477272727272727, "Indonesian,Vietnamese": 0.6363636363636364, "English,Filipino": 0.48863636363636365, "English,Spanish": 0.6136363636363636, "English,Chinese": 0.5397727272727273, "English,Malay": 0.6079545454545454, "English,Vietnamese": 0.5681818181818182, "Filipino,Spanish": 0.5284090909090909, "Filipino,Chinese": 0.5170454545454546, "Filipino,Malay": 0.5227272727272727, "Filipino,Vietnamese": 0.5340909090909091, "Spanish,Chinese": 0.5852272727272727, "Spanish,Malay": 0.5795454545454546, "Spanish,Vietnamese": 0.6136363636363636, "Chinese,Malay": 0.5454545454545454, "Chinese,Vietnamese": 0.6136363636363636, "Malay,Vietnamese": 0.625 }, "3_combine": { "Indonesian,English,Filipino": 0.32954545454545453, "Indonesian,English,Spanish": 0.4147727272727273, "Indonesian,English,Chinese": 0.36363636363636365, "Indonesian,English,Malay": 0.42613636363636365, "Indonesian,English,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish": 0.375, "Indonesian,Filipino,Chinese": 0.3181818181818182, "Indonesian,Filipino,Malay": 0.4034090909090909, "Indonesian,Filipino,Vietnamese": 0.4034090909090909, "Indonesian,Spanish,Chinese": 0.4090909090909091, "Indonesian,Spanish,Malay": 0.4375, "Indonesian,Spanish,Vietnamese": 0.4431818181818182, "Indonesian,Chinese,Malay": 0.39204545454545453, "Indonesian,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,Malay,Vietnamese": 0.48863636363636365, "English,Filipino,Spanish": 0.36363636363636365, "English,Filipino,Chinese": 0.3409090909090909, "English,Filipino,Malay": 0.35795454545454547, "English,Filipino,Vietnamese": 0.3465909090909091, "English,Spanish,Chinese": 0.4090909090909091, "English,Spanish,Malay": 0.4375, "English,Spanish,Vietnamese": 0.42045454545454547, "English,Chinese,Malay": 0.38636363636363635, "English,Chinese,Vietnamese": 0.3977272727272727, "English,Malay,Vietnamese": 0.4375, "Filipino,Spanish,Chinese": 0.35795454545454547, "Filipino,Spanish,Malay": 0.375, "Filipino,Spanish,Vietnamese": 0.3693181818181818, "Filipino,Chinese,Malay": 0.3465909090909091, "Filipino,Chinese,Vietnamese": 0.36363636363636365, "Filipino,Malay,Vietnamese": 0.38636363636363635, "Spanish,Chinese,Malay": 0.4034090909090909, "Spanish,Chinese,Vietnamese": 0.4375, "Spanish,Malay,Vietnamese": 0.4375, "Chinese,Malay,Vietnamese": 0.42045454545454547 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2727272727272727, "Indonesian,English,Filipino,Chinese": 0.24431818181818182, "Indonesian,English,Filipino,Malay": 0.29545454545454547, "Indonesian,English,Filipino,Vietnamese": 0.2840909090909091, "Indonesian,English,Spanish,Chinese": 0.3125, "Indonesian,English,Spanish,Malay": 0.3409090909090909, "Indonesian,English,Spanish,Vietnamese": 0.3409090909090909, "Indonesian,English,Chinese,Malay": 0.2897727272727273, "Indonesian,English,Chinese,Vietnamese": 0.3181818181818182, "Indonesian,English,Malay,Vietnamese": 0.3522727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Malay": 0.3125, "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, "Indonesian,Filipino,Chinese,Malay": 0.26704545454545453, "Indonesian,Filipino,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Malay,Vietnamese": 0.32954545454545453, "Indonesian,Spanish,Chinese,Malay": 0.3068181818181818, "Indonesian,Spanish,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.35795454545454547, "Indonesian,Chinese,Malay,Vietnamese": 0.32954545454545453, "English,Filipino,Spanish,Chinese": 0.26704545454545453, "English,Filipino,Spanish,Malay": 0.2897727272727273, "English,Filipino,Spanish,Vietnamese": 0.26704545454545453, "English,Filipino,Chinese,Malay": 0.2556818181818182, "English,Filipino,Chinese,Vietnamese": 0.2556818181818182, "English,Filipino,Malay,Vietnamese": 0.2840909090909091, "English,Spanish,Chinese,Malay": 0.3125, "English,Spanish,Chinese,Vietnamese": 0.3181818181818182, "English,Spanish,Malay,Vietnamese": 0.3409090909090909, "English,Chinese,Malay,Vietnamese": 0.3181818181818182, "Filipino,Spanish,Chinese,Malay": 0.2784090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.3068181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.26704545454545453, "Spanish,Chinese,Malay,Vietnamese": 0.32386363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2159090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.25, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.21022727272727273, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.25, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, "English,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 } }, "AC3_2": 0.5293809699466722, "AC3_3": 0.43971431407252337, "AC3_4": 0.37387357535718646, "AC3_5": 0.3237737804234065, "AC3_6": 0.2862289225514595, "AC3_7": 0.2598384082277527 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6990291262135923 } }, "cn_eval": { "prompt_1": { "accuracy": 0.42857142857142855 } }, "us_eval": { "prompt_1": { "accuracy": 0.7102803738317757 } }, "ph_eval": { "prompt_1": { "accuracy": 0.6, "category_acc": { "brand": 0.7, "demographics": 0.2, "biology": 0.7, "history": 0.6666666666666666, "literature": 0.4, "politics": 0.9, "culture": 0.7, "film": 0.5, "law": 0.4, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.35830522757252475 } }, "indommlu": { "prompt_1": { "accuracy": 0.5063088323653114, "category_acc": { "History": 0.5040160642570282, "Geography": 0.46122448979591835, "Lampungic": 0.3469387755102041, "Social science": 0.7395659432387313, "Balinese": 0.30148619957537154, "Makassarese": 0.3387096774193548, "Banjarese": 0.4444444444444444, "Chemistry": 0.35912408759124087, "Biology": 0.49230769230769234, "Science": 0.630546955624355, "Christian religion": 0.6368159203980099, "Art": 0.589018302828619, "Islam religion": 0.6088193456614509, "Hindu religion": 0.5066666666666667, "Madurese": 0.31186440677966104, "Sport": 0.5472972972972973, "Indonesian language": 0.5585305105853051, "Physics": 0.40606060606060607, "Minangkabau culture": 0.4321608040201005, "Dayak language": 0.3394495412844037, "Sociology": 0.4879032258064516, "Economy": 0.4651639344262295, "Sundanese": 0.4373379429559205, "Javanese": 0.39919354838709675, "Civic education": 0.6051502145922747 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.40201822328688375 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.33775261020094416 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.2651898327526456 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.41702568484678204 } }, "mmlu": { "prompt_1": { "accuracy": 0.6114352392065344 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.6112263139077583, "category_acc": { "high_school_european_history": 0.7012195121951219, "business_ethics": 0.6464646464646465, "clinical_knowledge": 0.696969696969697, "medical_genetics": 0.7575757575757576, "high_school_us_history": 0.7733990147783252, "high_school_physics": 0.4533333333333333, "high_school_world_history": 0.8135593220338984, "virology": 0.503030303030303, "high_school_microeconomics": 0.729957805907173, "econometrics": 0.45132743362831856, "college_computer_science": 0.47474747474747475, "high_school_biology": 0.7572815533980582, "abstract_algebra": 0.37373737373737376, "professional_accounting": 0.5124555160142349, "philosophy": 0.6741935483870968, "professional_medicine": 0.7047970479704797, "nutrition": 0.7213114754098361, "global_facts": 0.41414141414141414, "machine_learning": 0.44144144144144143, "security_studies": 0.6434426229508197, "public_relations": 0.6697247706422018, "professional_psychology": 0.6710310965630114, "prehistory": 0.7089783281733746, "anatomy": 0.664179104477612, "human_sexuality": 0.7307692307692307, "college_medicine": 0.627906976744186, "high_school_government_and_politics": 0.859375, "college_chemistry": 0.37373737373737376, "logical_fallacies": 0.7283950617283951, "high_school_geography": 0.7868020304568528, "elementary_mathematics": 0.40848806366047746, "human_aging": 0.6981981981981982, "college_mathematics": 0.3333333333333333, "high_school_psychology": 0.8051470588235294, "formal_logic": 0.392, "high_school_statistics": 0.49767441860465117, "international_law": 0.7666666666666667, "high_school_mathematics": 0.3754646840148699, "high_school_computer_science": 0.6767676767676768, "conceptual_physics": 0.5299145299145299, "miscellaneous": 0.7928388746803069, "high_school_chemistry": 0.5247524752475248, "marketing": 0.8412017167381974, "professional_law": 0.4474885844748858, "management": 0.8333333333333334, "college_physics": 0.4752475247524752, "jurisprudence": 0.7383177570093458, "world_religions": 0.8058823529411765, "sociology": 0.8, "us_foreign_policy": 0.8282828282828283, "high_school_macroeconomics": 0.6041131105398457, "computer_security": 0.7474747474747475, "moral_scenarios": 0.2483221476510067, "moral_disputes": 0.6753623188405797, "electrical_engineering": 0.5902777777777778, "astronomy": 0.695364238410596, "college_biology": 0.7342657342657343 } } }, "c_eval": { "prompt_1": { "accuracy": 0.4799405646359584 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.4900373599003736, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5833333333333334, "computer_architecture": 0.46153846153846156, "college_programming": 0.47619047619047616, "college_physics": 0.4583333333333333, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.35714285714285715, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.375, "middle_school_biology": 0.8076923076923077, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.64, "veterinary_medicine": 0.4642857142857143, "college_economics": 0.43333333333333335, "business_administration": 0.5526315789473685, "marxism": 0.5, "mao_zedong_thought": 0.5172413793103449, "education_science": 0.5588235294117647, "teacher_qualification": 0.6938775510204082, "high_school_politics": 0.6666666666666666, "high_school_geography": 0.625, "middle_school_politics": 0.8076923076923077, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.5555555555555556, "law": 0.5172413793103449, "chinese_language_and_literature": 0.42857142857142855, "art_studies": 0.5526315789473685, "professional_tour_guide": 0.5, "legal_professional": 0.5714285714285714, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.56, "middle_school_history": 0.7037037037037037, "civil_servant": 0.4807692307692308, "sports_science": 0.2916666666666667, "plant_protection": 0.6666666666666666, "basic_medicine": 0.375, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.5294117647058824, "accountant": 0.5555555555555556, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.5277777777777778, "tax_accountant": 0.3888888888888889, "physician": 0.5370370370370371 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.5161290322580645 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.4803142807805215, "category_acc": { "agronomy": 0.4556213017751479, "anatomy": 0.32432432432432434, "ancient_chinese": 0.29878048780487804, "arts": 0.58125, "astronomy": 0.2909090909090909, "business_ethics": 0.5550239234449761, "chinese_civil_service_exam": 0.49375, "chinese_driving_rule": 0.648854961832061, "chinese_food_culture": 0.38235294117647056, "chinese_foreign_policy": 0.4953271028037383, "chinese_history": 0.544891640866873, "chinese_literature": 0.37254901960784315, "chinese_teacher_qualification": 0.5865921787709497, "clinical_knowledge": 0.4177215189873418, "college_actuarial_science": 0.2358490566037736, "college_education": 0.5514018691588785, "college_engineering_hydrology": 0.44339622641509435, "college_law": 0.39814814814814814, "college_mathematics": 0.3619047619047619, "college_medical_statistics": 0.5094339622641509, "college_medicine": 0.38461538461538464, "computer_science": 0.553921568627451, "computer_security": 0.6257309941520468, "conceptual_physics": 0.48299319727891155, "construction_project_management": 0.3669064748201439, "economics": 0.5723270440251572, "education": 0.50920245398773, "electrical_engineering": 0.4883720930232558, "elementary_chinese": 0.3531746031746032, "elementary_commonsense": 0.45454545454545453, "elementary_information_and_technology": 0.7478991596638656, "elementary_mathematics": 0.3, "ethnology": 0.3925925925925926, "food_science": 0.5874125874125874, "genetics": 0.4147727272727273, "global_facts": 0.5704697986577181, "high_school_biology": 0.44970414201183434, "high_school_chemistry": 0.3484848484848485, "high_school_geography": 0.4576271186440678, "high_school_mathematics": 0.27439024390243905, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.45454545454545453, "human_sexuality": 0.5079365079365079, "international_law": 0.4486486486486487, "journalism": 0.5116279069767442, "jurisprudence": 0.5328467153284672, "legal_and_moral_basis": 0.8130841121495327, "logical": 0.5284552845528455, "machine_learning": 0.5409836065573771, "management": 0.580952380952381, "marketing": 0.5555555555555556, "marxist_theory": 0.544973544973545, "modern_chinese": 0.3275862068965517, "nutrition": 0.4482758620689655, "philosophy": 0.5904761904761905, "professional_accounting": 0.5085714285714286, "professional_law": 0.3791469194312796, "professional_medicine": 0.3723404255319149, "professional_psychology": 0.4956896551724138, "public_relations": 0.5114942528735632, "security_study": 0.6148148148148148, "sociology": 0.5353982300884956, "sports_science": 0.4727272727272727, "traditional_chinese_medicine": 0.2864864864864865, "virology": 0.6035502958579881, "world_history": 0.5838509316770186, "world_religions": 0.61875 } } }, "zbench": { "prompt_1": { "accuracy": 0.36363636363636365 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.44545454545454544 } }, "ocnli": { "prompt_1": { "accuracy": 0.40372881355932205 } }, "c3": { "prompt_1": { "accuracy": 0.8130142109199701 } }, "dream": { "prompt_1": { "accuracy": 0.8593826555609995 } }, "samsum": { "prompt_1": { "rouge1": 0.46593725641472133, "rouge2": 0.2267578192605436, "rougeL": 0.3814517016374434, "avg_rouge": 0.3580489257709028 } }, "dialogsum": { "prompt_1": { "rouge1": 0.42405246791568507, "rouge2": 0.17569185972744117, "rougeL": 0.3491933150322079, "avg_rouge": 0.31631254755844473 } }, "sst2": { "prompt_1": { "accuracy": 0.7121559633027523 } }, "cola": { "prompt_1": { "accuracy": 0.673058485139022 } }, "qqp": { "prompt_1": { "accuracy": 0.573 } }, "mnli": { "prompt_1": { "accuracy": 0.4 } }, "qnli": { "prompt_1": { "accuracy": 0.6 } }, "wnli": { "prompt_1": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.6 } }, "mrpc": { "prompt_1": { "accuracy": 0.6 } } } }, "mistral_7b_instruct_v0_2": { "model_size": "7B", "model_link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.8600840336134454, "language_acc": { "Spanish": 0.8798319327731092, "English": 0.8966386554621849, "Chinese": 0.8369747899159664, "Vietnamese": 0.826890756302521 }, "consistency_score_2": 0.8292717086834734, "consistency_score_3": 0.7571428571428571, "consistency_score_4": 0.7058823529411765, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8932773109243698, "Spanish,Chinese": 0.8285714285714286, "Spanish,Vietnamese": 0.8134453781512605, "English,Chinese": 0.8394957983193277, "English,Vietnamese": 0.8117647058823529, "Chinese,Vietnamese": 0.7890756302521008 }, "3_combine": { "Spanish,English,Chinese": 0.7907563025210084, "Spanish,English,Vietnamese": 0.7714285714285715, "Spanish,Chinese,Vietnamese": 0.7302521008403361, "English,Chinese,Vietnamese": 0.7361344537815127 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7058823529411765 } }, "AC3_2": 0.8443968765915703, "AC3_3": 0.8053371932842782, "AC3_4": 0.775391025692684 }, "prompt_2": { "overall_acc": 0.8619747899159664, "language_acc": { "Spanish": 0.880672268907563, "English": 0.9008403361344538, "Chinese": 0.8478991596638655, "Vietnamese": 0.8184873949579832 }, "consistency_score_2": 0.8357142857142857, "consistency_score_3": 0.7651260504201681, "consistency_score_4": 0.7184873949579832, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8915966386554622, "Spanish,Chinese": 0.8436974789915966, "Spanish,Vietnamese": 0.8117647058823529, "English,Chinese": 0.8546218487394958, "English,Vietnamese": 0.8100840336134454, "Chinese,Vietnamese": 0.8025210084033614 }, "3_combine": { "Spanish,English,Chinese": 0.8033613445378152, "Spanish,English,Vietnamese": 0.7663865546218488, "Spanish,Chinese,Vietnamese": 0.7428571428571429, "English,Chinese,Vietnamese": 0.7478991596638656 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7184873949579832 } }, "AC3_2": 0.8486414340016675, "AC3_3": 0.8106680915896736, "AC3_4": 0.7837176076900385 }, "prompt_3": { "overall_acc": 0.8609243697478992, "language_acc": { "Spanish": 0.8815126050420168, "English": 0.8983193277310925, "Chinese": 0.8411764705882353, "Vietnamese": 0.8226890756302521 }, "consistency_score_2": 0.8336134453781513, "consistency_score_3": 0.7628151260504201, "consistency_score_4": 0.7176470588235294, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8890756302521008, "Spanish,Chinese": 0.8369747899159664, "Spanish,Vietnamese": 0.8176470588235294, "English,Chinese": 0.8445378151260504, "English,Vietnamese": 0.8117647058823529, "Chinese,Vietnamese": 0.8016806722689076 }, "3_combine": { "Spanish,English,Chinese": 0.7941176470588235, "Spanish,English,Vietnamese": 0.7697478991596639, "Spanish,Chinese,Vietnamese": 0.7436974789915967, "English,Chinese,Vietnamese": 0.7436974789915967 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7176470588235294 } }, "AC3_2": 0.8470488219583601, "AC3_3": 0.8089057798840994, "AC3_4": 0.782783509947437 }, "prompt_4": { "overall_acc": 0.8581932773109244, "language_acc": { "Spanish": 0.8865546218487395, "English": 0.8899159663865546, "Chinese": 0.8361344537815126, "Vietnamese": 0.8201680672268907 }, "consistency_score_2": 0.8315126050420169, "consistency_score_3": 0.759453781512605, "consistency_score_4": 0.711764705882353, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8865546218487395, "Spanish,Chinese": 0.8420168067226891, "Spanish,Vietnamese": 0.8176470588235294, "English,Chinese": 0.8344537815126051, "English,Vietnamese": 0.8050420168067227, "Chinese,Vietnamese": 0.8033613445378152 }, "3_combine": { "Spanish,English,Chinese": 0.7924369747899159, "Spanish,English,Vietnamese": 0.7647058823529411, "Spanish,Chinese,Vietnamese": 0.7436974789915967, "English,Chinese,Vietnamese": 0.7369747899159664 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.711764705882353 } }, "AC3_2": 0.8446422955105344, "AC3_3": 0.8058100512434309, "AC3_4": 0.7781503608575436 }, "prompt_5": { "overall_acc": 0.8556722689075631, "language_acc": { "Spanish": 0.8798319327731092, "English": 0.892436974789916, "Chinese": 0.8285714285714286, "Vietnamese": 0.8218487394957983 }, "consistency_score_2": 0.8245098039215687, "consistency_score_3": 0.7510504201680672, "consistency_score_4": 0.7008403361344537, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8915966386554622, "Spanish,Chinese": 0.8159663865546218, "Spanish,Vietnamese": 0.8058823529411765, "English,Chinese": 0.8302521008403362, "English,Vietnamese": 0.8134453781512605, "Chinese,Vietnamese": 0.7899159663865546 }, "3_combine": { "Spanish,English,Chinese": 0.7798319327731092, "Spanish,English,Vietnamese": 0.7663865546218488, "Spanish,Chinese,Vietnamese": 0.7260504201680672, "English,Chinese,Vietnamese": 0.7319327731092437 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7008403361344537 } }, "AC3_2": 0.8398020500577692, "AC3_3": 0.7999551153645786, "AC3_4": 0.77055545657774 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.4942857142857142, "language_acc": { "Filipino": 0.4533333333333333, "Vietnamese": 0.48, "Chinese": 0.4666666666666667, "Spanish": 0.5933333333333334, "Malay": 0.36666666666666664, "Indonesian": 0.4533333333333333, "English": 0.6466666666666666 }, "consistency_score_2": 0.4787301587301588, "consistency_score_3": 0.29599999999999993, "consistency_score_4": 0.20761904761904762, "consistency_score_5": 0.15682539682539684, "consistency_score_6": 0.1238095238095238, "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4066666666666667, "Filipino,Chinese": 0.42, "Filipino,Spanish": 0.44666666666666666, "Filipino,Malay": 0.49333333333333335, "Filipino,Indonesian": 0.41333333333333333, "Filipino,English": 0.5333333333333333, "Vietnamese,Chinese": 0.48, "Vietnamese,Spanish": 0.48, "Vietnamese,Malay": 0.52, "Vietnamese,Indonesian": 0.44, "Vietnamese,English": 0.4666666666666667, "Chinese,Spanish": 0.49333333333333335, "Chinese,Malay": 0.4533333333333333, "Chinese,Indonesian": 0.4066666666666667, "Chinese,English": 0.5066666666666667, "Spanish,Malay": 0.5066666666666667, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.5666666666666667, "Malay,Indonesian": 0.5266666666666666, "Malay,English": 0.49333333333333335, "Indonesian,English": 0.5066666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.24666666666666667, "Filipino,Vietnamese,Spanish": 0.26, "Filipino,Vietnamese,Malay": 0.2866666666666667, "Filipino,Vietnamese,Indonesian": 0.22, "Filipino,Vietnamese,English": 0.2866666666666667, "Filipino,Chinese,Spanish": 0.26666666666666666, "Filipino,Chinese,Malay": 0.26, "Filipino,Chinese,Indonesian": 0.23333333333333334, "Filipino,Chinese,English": 0.3, "Filipino,Spanish,Malay": 0.31333333333333335, "Filipino,Spanish,Indonesian": 0.25333333333333335, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.30666666666666664, "Filipino,Malay,English": 0.32666666666666666, "Filipino,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish": 0.31333333333333335, "Vietnamese,Chinese,Malay": 0.32666666666666666, "Vietnamese,Chinese,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,English": 0.31333333333333335, "Vietnamese,Spanish,Malay": 0.3333333333333333, "Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,English": 0.32, "Vietnamese,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Malay,English": 0.31333333333333335, "Vietnamese,Indonesian,English": 0.2733333333333333, "Chinese,Spanish,Malay": 0.30666666666666664, "Chinese,Spanish,Indonesian": 0.2733333333333333, "Chinese,Spanish,English": 0.35333333333333333, "Chinese,Malay,Indonesian": 0.2733333333333333, "Chinese,Malay,English": 0.3, "Chinese,Indonesian,English": 0.28, "Spanish,Malay,Indonesian": 0.31333333333333335, "Spanish,Malay,English": 0.35333333333333333, "Spanish,Indonesian,English": 0.36, "Malay,Indonesian,English": 0.34 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Malay,English": 0.22, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.20666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Chinese,Spanish,English": 0.21333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.18666666666666668, "Filipino,Chinese,Malay,English": 0.21333333333333335, "Filipino,Chinese,Indonesian,English": 0.18666666666666668, "Filipino,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Spanish,Malay,English": 0.24, "Filipino,Spanish,Indonesian,English": 0.22, "Filipino,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,Spanish,English": 0.24666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.2, "Vietnamese,Chinese,Malay,English": 0.23333333333333334, "Vietnamese,Chinese,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Vietnamese,Spanish,Malay,English": 0.23333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Vietnamese,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Chinese,Spanish,Malay,English": 0.24, "Chinese,Spanish,Indonesian,English": 0.22, "Chinese,Malay,Indonesian,English": 0.22, "Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.16, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Malay,Indonesian,English": 0.16, "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.18 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, "AC3_2": 0.48638359351794563, "AC3_3": 0.370267534298773, "AC3_4": 0.2924132583417477, "AC3_5": 0.2381054537490407, "AC3_6": 0.19801893019028305, "AC3_7": 0.16634615381816292 }, "prompt_2": { "overall_acc": 0.4933333333333333, "language_acc": { "Filipino": 0.4666666666666667, "Vietnamese": 0.47333333333333333, "Chinese": 0.3933333333333333, "Spanish": 0.56, "Malay": 0.38, "Indonesian": 0.5133333333333333, "English": 0.6666666666666666 }, "consistency_score_2": 0.49523809523809537, "consistency_score_3": 0.31752380952380943, "consistency_score_4": 0.2287619047619048, "consistency_score_5": 0.1736507936507937, "consistency_score_6": 0.13523809523809524, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4, "Filipino,Chinese": 0.38, "Filipino,Spanish": 0.52, "Filipino,Malay": 0.43333333333333335, "Filipino,Indonesian": 0.44666666666666666, "Filipino,English": 0.47333333333333333, "Vietnamese,Chinese": 0.4266666666666667, "Vietnamese,Spanish": 0.5266666666666666, "Vietnamese,Malay": 0.5133333333333333, "Vietnamese,Indonesian": 0.52, "Vietnamese,English": 0.5, "Chinese,Spanish": 0.5133333333333333, "Chinese,Malay": 0.46, "Chinese,Indonesian": 0.52, "Chinese,English": 0.42, "Spanish,Malay": 0.56, "Spanish,Indonesian": 0.54, "Spanish,English": 0.6533333333333333, "Malay,Indonesian": 0.56, "Malay,English": 0.47333333333333333, "Indonesian,English": 0.56 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.21333333333333335, "Filipino,Vietnamese,Spanish": 0.2733333333333333, "Filipino,Vietnamese,Malay": 0.26666666666666666, "Filipino,Vietnamese,Indonesian": 0.28, "Filipino,Vietnamese,English": 0.24666666666666667, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.24, "Filipino,Chinese,Indonesian": 0.2733333333333333, "Filipino,Chinese,English": 0.22666666666666666, "Filipino,Spanish,Malay": 0.34, "Filipino,Spanish,Indonesian": 0.32, "Filipino,Spanish,English": 0.38, "Filipino,Malay,Indonesian": 0.2866666666666667, "Filipino,Malay,English": 0.2733333333333333, "Filipino,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish": 0.30666666666666664, "Vietnamese,Chinese,Malay": 0.3, "Vietnamese,Chinese,Indonesian": 0.30666666666666664, "Vietnamese,Chinese,English": 0.26, "Vietnamese,Spanish,Malay": 0.36, "Vietnamese,Spanish,Indonesian": 0.36666666666666664, "Vietnamese,Spanish,English": 0.38666666666666666, "Vietnamese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Malay,English": 0.32, "Vietnamese,Indonesian,English": 0.38, "Chinese,Spanish,Malay": 0.3466666666666667, "Chinese,Spanish,Indonesian": 0.3466666666666667, "Chinese,Spanish,English": 0.34, "Chinese,Malay,Indonesian": 0.32666666666666666, "Chinese,Malay,English": 0.28, "Chinese,Indonesian,English": 0.31333333333333335, "Spanish,Malay,Indonesian": 0.38666666666666666, "Spanish,Malay,English": 0.4, "Spanish,Indonesian,English": 0.43333333333333335, "Malay,Indonesian,English": 0.37333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Chinese,English": 0.16, "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.20666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.2, "Filipino,Chinese,Malay,English": 0.17333333333333334, "Filipino,Chinese,Indonesian,English": 0.18666666666666668, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.26, "Filipino,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,Spanish,English": 0.23333333333333334, "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Malay,English": 0.20666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.28, "Vietnamese,Spanish,Malay,English": 0.2733333333333333, "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, "Vietnamese,Malay,Indonesian,English": 0.2733333333333333, "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, "Chinese,Spanish,Malay,English": 0.25333333333333335, "Chinese,Spanish,Indonesian,English": 0.26, "Chinese,Malay,Indonesian,English": 0.24, "Spanish,Malay,Indonesian,English": 0.32 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18, "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, "Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.49428387920497774, "AC3_3": 0.3863691174842586, "AC3_4": 0.3125789149307736, "AC3_5": 0.25688085035019004, "AC3_6": 0.212282828249056, "AC3_7": 0.17540740737817284 }, "prompt_3": { "overall_acc": 0.4942857142857143, "language_acc": { "Filipino": 0.4666666666666667, "Vietnamese": 0.46, "Chinese": 0.4066666666666667, "Spanish": 0.58, "Malay": 0.3933333333333333, "Indonesian": 0.49333333333333335, "English": 0.66 }, "consistency_score_2": 0.4984126984126984, "consistency_score_3": 0.32095238095238093, "consistency_score_4": 0.23466666666666672, "consistency_score_5": 0.18476190476190477, "consistency_score_6": 0.15142857142857144, "consistency_score_7": 0.12666666666666668, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.46, "Filipino,Chinese": 0.4066666666666667, "Filipino,Spanish": 0.49333333333333335, "Filipino,Malay": 0.5066666666666667, "Filipino,Indonesian": 0.5133333333333333, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.4, "Vietnamese,Spanish": 0.4866666666666667, "Vietnamese,Malay": 0.54, "Vietnamese,Indonesian": 0.5066666666666667, "Vietnamese,English": 0.44666666666666666, "Chinese,Spanish": 0.4866666666666667, "Chinese,Malay": 0.5066666666666667, "Chinese,Indonesian": 0.5333333333333333, "Chinese,English": 0.44, "Spanish,Malay": 0.5066666666666667, "Spanish,Indonesian": 0.56, "Spanish,English": 0.6733333333333333, "Malay,Indonesian": 0.5066666666666667, "Malay,English": 0.47333333333333333, "Indonesian,English": 0.5266666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.22666666666666666, "Filipino,Vietnamese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Malay": 0.32, "Filipino,Vietnamese,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,English": 0.2866666666666667, "Filipino,Chinese,Spanish": 0.28, "Filipino,Chinese,Malay": 0.28, "Filipino,Chinese,Indonesian": 0.3, "Filipino,Chinese,English": 0.26666666666666666, "Filipino,Spanish,Malay": 0.32666666666666666, "Filipino,Spanish,Indonesian": 0.3333333333333333, "Filipino,Spanish,English": 0.38, "Filipino,Malay,Indonesian": 0.3333333333333333, "Filipino,Malay,English": 0.32666666666666666, "Filipino,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish": 0.26666666666666666, "Vietnamese,Chinese,Malay": 0.31333333333333335, "Vietnamese,Chinese,Indonesian": 0.2866666666666667, "Vietnamese,Chinese,English": 0.24, "Vietnamese,Spanish,Malay": 0.34, "Vietnamese,Spanish,Indonesian": 0.3333333333333333, "Vietnamese,Spanish,English": 0.35333333333333333, "Vietnamese,Malay,Indonesian": 0.35333333333333333, "Vietnamese,Malay,English": 0.30666666666666664, "Vietnamese,Indonesian,English": 0.32, "Chinese,Spanish,Malay": 0.32666666666666666, "Chinese,Spanish,Indonesian": 0.35333333333333333, "Chinese,Spanish,English": 0.36, "Chinese,Malay,Indonesian": 0.3333333333333333, "Chinese,Malay,English": 0.29333333333333333, "Chinese,Indonesian,English": 0.32666666666666666, "Spanish,Malay,Indonesian": 0.35333333333333333, "Spanish,Malay,English": 0.3933333333333333, "Spanish,Indonesian,English": 0.4266666666666667, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,English": 0.18, "Filipino,Vietnamese,Spanish,Malay": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.24, "Filipino,Vietnamese,Spanish,English": 0.24, "Filipino,Vietnamese,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.24, "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.23333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.21333333333333335, "Filipino,Chinese,Malay,English": 0.22666666666666666, "Filipino,Chinese,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian": 0.26, "Filipino,Spanish,Malay,English": 0.2733333333333333, "Filipino,Spanish,Indonesian,English": 0.28, "Filipino,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Malay,English": 0.2, "Vietnamese,Chinese,Indonesian,English": 0.19333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, "Vietnamese,Spanish,Malay,English": 0.25333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, "Vietnamese,Malay,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, "Chinese,Spanish,Malay,English": 0.26666666666666666, "Chinese,Spanish,Indonesian,English": 0.28, "Chinese,Malay,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.16, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.18, "Filipino,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,English": 0.18, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.16666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 } }, "AC3_2": 0.4963406276625506, "AC3_3": 0.3891922562940493, "AC3_4": 0.31824405535221373, "AC3_5": 0.2689801642561209, "AC3_6": 0.23183312259367947, "AC3_7": 0.20165644168531607 }, "prompt_4": { "overall_acc": 0.4847619047619048, "language_acc": { "Filipino": 0.4666666666666667, "Vietnamese": 0.4266666666666667, "Chinese": 0.4266666666666667, "Spanish": 0.5333333333333333, "Malay": 0.36666666666666664, "Indonesian": 0.52, "English": 0.6533333333333333 }, "consistency_score_2": 0.4809523809523809, "consistency_score_3": 0.29847619047619056, "consistency_score_4": 0.2100952380952381, "consistency_score_5": 0.1596825396825397, "consistency_score_6": 0.12761904761904763, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.38666666666666666, "Filipino,Chinese": 0.38, "Filipino,Spanish": 0.5133333333333333, "Filipino,Malay": 0.4266666666666667, "Filipino,Indonesian": 0.4533333333333333, "Filipino,English": 0.5066666666666667, "Vietnamese,Chinese": 0.44, "Vietnamese,Spanish": 0.47333333333333333, "Vietnamese,Malay": 0.49333333333333335, "Vietnamese,Indonesian": 0.5466666666666666, "Vietnamese,English": 0.47333333333333333, "Chinese,Spanish": 0.47333333333333333, "Chinese,Malay": 0.41333333333333333, "Chinese,Indonesian": 0.4533333333333333, "Chinese,English": 0.48, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.54, "Spanish,English": 0.62, "Malay,Indonesian": 0.5533333333333333, "Malay,English": 0.4266666666666667, "Indonesian,English": 0.56 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.23333333333333334, "Filipino,Vietnamese,Spanish": 0.2733333333333333, "Filipino,Vietnamese,Malay": 0.24, "Filipino,Vietnamese,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,English": 0.2733333333333333, "Filipino,Chinese,Spanish": 0.2733333333333333, "Filipino,Chinese,Malay": 0.21333333333333335, "Filipino,Chinese,Indonesian": 0.25333333333333335, "Filipino,Chinese,English": 0.25333333333333335, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.38666666666666666, "Filipino,Malay,Indonesian": 0.2866666666666667, "Filipino,Malay,English": 0.28, "Filipino,Indonesian,English": 0.32, "Vietnamese,Chinese,Spanish": 0.2733333333333333, "Vietnamese,Chinese,Malay": 0.26666666666666666, "Vietnamese,Chinese,Indonesian": 0.30666666666666664, "Vietnamese,Chinese,English": 0.28, "Vietnamese,Spanish,Malay": 0.32, "Vietnamese,Spanish,Indonesian": 0.3333333333333333, "Vietnamese,Spanish,English": 0.32666666666666666, "Vietnamese,Malay,Indonesian": 0.35333333333333333, "Vietnamese,Malay,English": 0.2733333333333333, "Vietnamese,Indonesian,English": 0.35333333333333333, "Chinese,Spanish,Malay": 0.2733333333333333, "Chinese,Spanish,Indonesian": 0.2866666666666667, "Chinese,Spanish,English": 0.34, "Chinese,Malay,Indonesian": 0.2866666666666667, "Chinese,Malay,English": 0.24666666666666667, "Chinese,Indonesian,English": 0.29333333333333333, "Spanish,Malay,Indonesian": 0.37333333333333335, "Spanish,Malay,English": 0.3333333333333333, "Spanish,Indonesian,English": 0.4, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18, "Filipino,Vietnamese,Chinese,Malay": 0.16, "Filipino,Vietnamese,Chinese,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Spanish,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay": 0.18, "Filipino,Chinese,Spanish,Indonesian": 0.19333333333333333, "Filipino,Chinese,Spanish,English": 0.21333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Malay,English": 0.16666666666666666, "Filipino,Chinese,Indonesian,English": 0.19333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.24666666666666667, "Filipino,Spanish,Indonesian,English": 0.26, "Filipino,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Spanish,English": 0.20666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Malay,English": 0.18, "Vietnamese,Chinese,Indonesian,English": 0.22, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.23333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, "Chinese,Spanish,Malay,English": 0.2, "Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Chinese,Malay,Indonesian,English": 0.19333333333333333, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.14, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian,English": 0.16, "Filipino,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, "Vietnamese,Chinese,Spanish,Malay,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, "Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.4828496289534759, "AC3_3": 0.36946590693819076, "AC3_4": 0.29314275267293366, "AC3_5": 0.24023176163290802, "AC3_6": 0.20204695249607396, "AC3_7": 0.1748577562774664 }, "prompt_5": { "overall_acc": 0.46380952380952384, "language_acc": { "Filipino": 0.41333333333333333, "Vietnamese": 0.4666666666666667, "Chinese": 0.42, "Spanish": 0.49333333333333335, "Malay": 0.3466666666666667, "Indonesian": 0.4666666666666667, "English": 0.64 }, "consistency_score_2": 0.45682539682539686, "consistency_score_3": 0.2666666666666666, "consistency_score_4": 0.18095238095238095, "consistency_score_5": 0.13714285714285715, "consistency_score_6": 0.11333333333333333, "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.41333333333333333, "Filipino,Chinese": 0.36666666666666664, "Filipino,Spanish": 0.46, "Filipino,Malay": 0.43333333333333335, "Filipino,Indonesian": 0.4666666666666667, "Filipino,English": 0.4866666666666667, "Vietnamese,Chinese": 0.4, "Vietnamese,Spanish": 0.44666666666666666, "Vietnamese,Malay": 0.42, "Vietnamese,Indonesian": 0.44, "Vietnamese,English": 0.44, "Chinese,Spanish": 0.5133333333333333, "Chinese,Malay": 0.37333333333333335, "Chinese,Indonesian": 0.46, "Chinese,English": 0.5066666666666667, "Spanish,Malay": 0.43333333333333335, "Spanish,Indonesian": 0.48, "Spanish,English": 0.5933333333333334, "Malay,Indonesian": 0.5133333333333333, "Malay,English": 0.44, "Indonesian,English": 0.5066666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2, "Filipino,Vietnamese,Spanish": 0.24, "Filipino,Vietnamese,Malay": 0.22, "Filipino,Vietnamese,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,English": 0.24666666666666667, "Filipino,Chinese,Spanish": 0.24666666666666667, "Filipino,Chinese,Malay": 0.19333333333333333, "Filipino,Chinese,Indonesian": 0.24666666666666667, "Filipino,Chinese,English": 0.24, "Filipino,Spanish,Malay": 0.25333333333333335, "Filipino,Spanish,Indonesian": 0.29333333333333333, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.31333333333333335, "Filipino,Malay,English": 0.25333333333333335, "Filipino,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish": 0.26, "Vietnamese,Chinese,Malay": 0.2, "Vietnamese,Chinese,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,English": 0.25333333333333335, "Vietnamese,Spanish,Malay": 0.25333333333333335, "Vietnamese,Spanish,Indonesian": 0.25333333333333335, "Vietnamese,Spanish,English": 0.29333333333333333, "Vietnamese,Malay,Indonesian": 0.26, "Vietnamese,Malay,English": 0.24666666666666667, "Vietnamese,Indonesian,English": 0.29333333333333333, "Chinese,Spanish,Malay": 0.24, "Chinese,Spanish,Indonesian": 0.3, "Chinese,Spanish,English": 0.36666666666666664, "Chinese,Malay,Indonesian": 0.26666666666666666, "Chinese,Malay,English": 0.24, "Chinese,Indonesian,English": 0.2866666666666667, "Spanish,Malay,Indonesian": 0.2866666666666667, "Spanish,Malay,English": 0.30666666666666664, "Spanish,Indonesian,English": 0.36666666666666664, "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay": 0.12, "Filipino,Vietnamese,Chinese,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,English": 0.14, "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.18, "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay": 0.16, "Filipino,Chinese,Spanish,Indonesian": 0.2, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.18, "Filipino,Chinese,Malay,English": 0.14, "Filipino,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Spanish,Malay,English": 0.21333333333333335, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Indonesian": 0.18, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.16, "Vietnamese,Chinese,Malay,English": 0.16666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian": 0.18, "Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Vietnamese,Spanish,Indonesian,English": 0.2, "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.18, "Chinese,Spanish,Malay,English": 0.19333333333333333, "Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Chinese,Malay,Indonesian,English": 0.17333333333333334, "Spanish,Malay,Indonesian,English": 0.23333333333333334 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.14, "Filipino,Chinese,Spanish,Indonesian,English": 0.16, "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.18, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,English": 0.14, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, "Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, "AC3_2": 0.46029096875131653, "AC3_3": 0.33863537587715326, "AC3_4": 0.26033621716437777, "AC3_5": 0.21169119308227463, "AC3_6": 0.18215621559000042, "AC3_7": 0.16452702699784572 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.4180194805194805, "language_acc": { "Indonesian": 0.4090909090909091, "English": 0.5340909090909091, "Filipino": 0.3409090909090909, "Spanish": 0.39204545454545453, "Chinese": 0.44886363636363635, "Malay": 0.3806818181818182, "Vietnamese": 0.42045454545454547 }, "consistency_score_2": 0.515151515151515, "consistency_score_3": 0.33084415584415583, "consistency_score_4": 0.23198051948051943, "consistency_score_5": 0.17288961038961037, "consistency_score_6": 0.13636363636363638, "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5568181818181818, "Indonesian,Filipino": 0.5170454545454546, "Indonesian,Spanish": 0.5681818181818182, "Indonesian,Chinese": 0.4715909090909091, "Indonesian,Malay": 0.5625, "Indonesian,Vietnamese": 0.5454545454545454, "English,Filipino": 0.4375, "English,Spanish": 0.5965909090909091, "English,Chinese": 0.5568181818181818, "English,Malay": 0.5568181818181818, "English,Vietnamese": 0.5340909090909091, "Filipino,Spanish": 0.4659090909090909, "Filipino,Chinese": 0.42613636363636365, "Filipino,Malay": 0.4602272727272727, "Filipino,Vietnamese": 0.45454545454545453, "Spanish,Chinese": 0.5511363636363636, "Spanish,Malay": 0.5568181818181818, "Spanish,Vietnamese": 0.5284090909090909, "Chinese,Malay": 0.48863636363636365, "Chinese,Vietnamese": 0.4943181818181818, "Malay,Vietnamese": 0.48863636363636365 }, "3_combine": { "Indonesian,English,Filipino": 0.3125, "Indonesian,English,Spanish": 0.4090909090909091, "Indonesian,English,Chinese": 0.3465909090909091, "Indonesian,English,Malay": 0.38636363636363635, "Indonesian,English,Vietnamese": 0.3693181818181818, "Indonesian,Filipino,Spanish": 0.3409090909090909, "Indonesian,Filipino,Chinese": 0.2727272727272727, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.32386363636363635, "Indonesian,Spanish,Chinese": 0.32954545454545453, "Indonesian,Spanish,Malay": 0.4034090909090909, "Indonesian,Spanish,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay": 0.3125, "Indonesian,Chinese,Vietnamese": 0.3125, "Indonesian,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish": 0.29545454545454547, "English,Filipino,Chinese": 0.2897727272727273, "English,Filipino,Malay": 0.2840909090909091, "English,Filipino,Vietnamese": 0.29545454545454547, "English,Spanish,Chinese": 0.39204545454545453, "English,Spanish,Malay": 0.39204545454545453, "English,Spanish,Vietnamese": 0.38636363636363635, "English,Chinese,Malay": 0.3693181818181818, "English,Chinese,Vietnamese": 0.3409090909090909, "English,Malay,Vietnamese": 0.3465909090909091, "Filipino,Spanish,Chinese": 0.2727272727272727, "Filipino,Spanish,Malay": 0.2897727272727273, "Filipino,Spanish,Vietnamese": 0.3125, "Filipino,Chinese,Malay": 0.25, "Filipino,Chinese,Vietnamese": 0.26704545454545453, "Filipino,Malay,Vietnamese": 0.2897727272727273, "Spanish,Chinese,Malay": 0.3465909090909091, "Spanish,Chinese,Vietnamese": 0.3352272727272727, "Spanish,Malay,Vietnamese": 0.3522727272727273, "Chinese,Malay,Vietnamese": 0.2897727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.23295454545454544, "Indonesian,English,Filipino,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Malay": 0.2159090909090909, "Indonesian,English,Filipino,Vietnamese": 0.23295454545454544, "Indonesian,English,Spanish,Chinese": 0.2784090909090909, "Indonesian,English,Spanish,Malay": 0.2897727272727273, "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.19318181818181818, "Indonesian,Filipino,Spanish,Malay": 0.25, "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Chinese,Malay": 0.19318181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Chinese,Malay": 0.24431818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.2897727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese": 0.19886363636363635, "English,Filipino,Spanish,Malay": 0.20454545454545456, "English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "English,Filipino,Chinese,Malay": 0.20454545454545456, "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, "English,Filipino,Malay,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese,Malay": 0.2840909090909091, "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, "English,Spanish,Malay,Vietnamese": 0.2784090909090909, "English,Chinese,Malay,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.21022727272727273, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2215909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.125, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, "AC3_2": 0.4615303513040302, "AC3_3": 0.3693577720290219, "AC3_4": 0.29837654224297583, "AC3_5": 0.24461030037247625, "AC3_6": 0.20564355114086408, "AC3_7": 0.17869535041746457 }, "prompt_2": { "overall_acc": 0.4480519480519481, "language_acc": { "Indonesian": 0.42613636363636365, "English": 0.5681818181818182, "Filipino": 0.3693181818181818, "Spanish": 0.4375, "Chinese": 0.4715909090909091, "Malay": 0.42045454545454547, "Vietnamese": 0.4431818181818182 }, "consistency_score_2": 0.525974025974026, "consistency_score_3": 0.3459415584415584, "consistency_score_4": 0.25129870129870124, "consistency_score_5": 0.19345238095238096, "consistency_score_6": 0.15584415584415587, "consistency_score_7": 0.13068181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5625, "Indonesian,Filipino": 0.5738636363636364, "Indonesian,Spanish": 0.625, "Indonesian,Chinese": 0.5170454545454546, "Indonesian,Malay": 0.6022727272727273, "Indonesian,Vietnamese": 0.5568181818181818, "English,Filipino": 0.44886363636363635, "English,Spanish": 0.625, "English,Chinese": 0.5795454545454546, "English,Malay": 0.4943181818181818, "English,Vietnamese": 0.4943181818181818, "Filipino,Spanish": 0.4715909090909091, "Filipino,Chinese": 0.45454545454545453, "Filipino,Malay": 0.5056818181818182, "Filipino,Vietnamese": 0.5113636363636364, "Spanish,Chinese": 0.5227272727272727, "Spanish,Malay": 0.5568181818181818, "Spanish,Vietnamese": 0.5511363636363636, "Chinese,Malay": 0.4318181818181818, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.4715909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.3522727272727273, "Indonesian,English,Spanish": 0.4431818181818182, "Indonesian,English,Chinese": 0.375, "Indonesian,English,Malay": 0.3693181818181818, "Indonesian,English,Vietnamese": 0.36363636363636365, "Indonesian,Filipino,Spanish": 0.375, "Indonesian,Filipino,Chinese": 0.3465909090909091, "Indonesian,Filipino,Malay": 0.3806818181818182, "Indonesian,Filipino,Vietnamese": 0.3806818181818182, "Indonesian,Spanish,Chinese": 0.375, "Indonesian,Spanish,Malay": 0.4318181818181818, "Indonesian,Spanish,Vietnamese": 0.4090909090909091, "Indonesian,Chinese,Malay": 0.3181818181818182, "Indonesian,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish": 0.32386363636363635, "English,Filipino,Chinese": 0.3181818181818182, "English,Filipino,Malay": 0.29545454545454547, "English,Filipino,Vietnamese": 0.3068181818181818, "English,Spanish,Chinese": 0.4147727272727273, "English,Spanish,Malay": 0.39204545454545453, "English,Spanish,Vietnamese": 0.38636363636363635, "English,Chinese,Malay": 0.3068181818181818, "English,Chinese,Vietnamese": 0.3465909090909091, "English,Malay,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Chinese": 0.2897727272727273, "Filipino,Spanish,Malay": 0.32386363636363635, "Filipino,Spanish,Vietnamese": 0.3465909090909091, "Filipino,Chinese,Malay": 0.26704545454545453, "Filipino,Chinese,Vietnamese": 0.29545454545454547, "Filipino,Malay,Vietnamese": 0.29545454545454547, "Spanish,Chinese,Malay": 0.3181818181818182, "Spanish,Chinese,Vietnamese": 0.3465909090909091, "Spanish,Malay,Vietnamese": 0.3465909090909091, "Chinese,Malay,Vietnamese": 0.2784090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2784090909090909, "Indonesian,English,Filipino,Chinese": 0.26136363636363635, "Indonesian,English,Filipino,Malay": 0.2556818181818182, "Indonesian,English,Filipino,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Chinese": 0.3068181818181818, "Indonesian,English,Spanish,Malay": 0.3068181818181818, "Indonesian,English,Spanish,Vietnamese": 0.30113636363636365, "Indonesian,English,Chinese,Malay": 0.23863636363636365, "Indonesian,English,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Spanish,Chinese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Malay": 0.2840909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Chinese,Malay": 0.22727272727272727, "Indonesian,Filipino,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Malay,Vietnamese": 0.25, "Indonesian,Spanish,Chinese,Malay": 0.25, "Indonesian,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish,Chinese": 0.24431818181818182, "English,Filipino,Spanish,Malay": 0.24431818181818182, "English,Filipino,Spanish,Vietnamese": 0.2556818181818182, "English,Filipino,Chinese,Malay": 0.2159090909090909, "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, "English,Filipino,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay": 0.26136363636363635, "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, "English,Spanish,Malay,Vietnamese": 0.26136363636363635, "English,Chinese,Malay,Vietnamese": 0.2215909090909091, "Filipino,Spanish,Chinese,Malay": 0.2159090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.21022727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.2159090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2159090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.1875, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,English,Filipino,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,English,Chinese,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1875, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.1875, "English,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, "English,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 } }, "AC3_2": 0.48389610384642395, "AC3_3": 0.39043087351398426, "AC3_4": 0.32199833587987, "AC3_5": 0.2702295595168321, "AC3_6": 0.2312526183110991, "AC3_7": 0.20234604102075487 }, "prompt_3": { "overall_acc": 0.4342532467532468, "language_acc": { "Indonesian": 0.4090909090909091, "English": 0.5568181818181818, "Filipino": 0.3465909090909091, "Spanish": 0.44886363636363635, "Chinese": 0.4602272727272727, "Malay": 0.4090909090909091, "Vietnamese": 0.4090909090909091 }, "consistency_score_2": 0.5300324675324676, "consistency_score_3": 0.35698051948051945, "consistency_score_4": 0.2662337662337662, "consistency_score_5": 0.21049783549783546, "consistency_score_6": 0.17370129870129872, "consistency_score_7": 0.14772727272727273, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5227272727272727, "Indonesian,Filipino": 0.5681818181818182, "Indonesian,Spanish": 0.6477272727272727, "Indonesian,Chinese": 0.48295454545454547, "Indonesian,Malay": 0.5795454545454546, "Indonesian,Vietnamese": 0.5681818181818182, "English,Filipino": 0.4318181818181818, "English,Spanish": 0.6306818181818182, "English,Chinese": 0.5454545454545454, "English,Malay": 0.5397727272727273, "English,Vietnamese": 0.4943181818181818, "Filipino,Spanish": 0.4943181818181818, "Filipino,Chinese": 0.44886363636363635, "Filipino,Malay": 0.48295454545454547, "Filipino,Vietnamese": 0.4715909090909091, "Spanish,Chinese": 0.5170454545454546, "Spanish,Malay": 0.6022727272727273, "Spanish,Vietnamese": 0.5909090909090909, "Chinese,Malay": 0.5170454545454546, "Chinese,Vietnamese": 0.4943181818181818, "Malay,Vietnamese": 0.5 }, "3_combine": { "Indonesian,English,Filipino": 0.3181818181818182, "Indonesian,English,Spanish": 0.44886363636363635, "Indonesian,English,Chinese": 0.32954545454545453, "Indonesian,English,Malay": 0.3693181818181818, "Indonesian,English,Vietnamese": 0.35795454545454547, "Indonesian,Filipino,Spanish": 0.4034090909090909, "Indonesian,Filipino,Chinese": 0.32386363636363635, "Indonesian,Filipino,Malay": 0.3693181818181818, "Indonesian,Filipino,Vietnamese": 0.36363636363636365, "Indonesian,Spanish,Chinese": 0.3806818181818182, "Indonesian,Spanish,Malay": 0.45454545454545453, "Indonesian,Spanish,Vietnamese": 0.45454545454545453, "Indonesian,Chinese,Malay": 0.3522727272727273, "Indonesian,Chinese,Vietnamese": 0.32386363636363635, "Indonesian,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish": 0.3352272727272727, "English,Filipino,Chinese": 0.29545454545454547, "English,Filipino,Malay": 0.2897727272727273, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.39204545454545453, "English,Spanish,Malay": 0.4375, "English,Spanish,Vietnamese": 0.42045454545454547, "English,Chinese,Malay": 0.36363636363636365, "English,Chinese,Vietnamese": 0.32954545454545453, "English,Malay,Vietnamese": 0.3409090909090909, "Filipino,Spanish,Chinese": 0.29545454545454547, "Filipino,Spanish,Malay": 0.3465909090909091, "Filipino,Spanish,Vietnamese": 0.3522727272727273, "Filipino,Chinese,Malay": 0.29545454545454547, "Filipino,Chinese,Vietnamese": 0.29545454545454547, "Filipino,Malay,Vietnamese": 0.3125, "Spanish,Chinese,Malay": 0.3693181818181818, "Spanish,Chinese,Vietnamese": 0.36363636363636365, "Spanish,Malay,Vietnamese": 0.3977272727272727, "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2897727272727273, "Indonesian,English,Filipino,Chinese": 0.2215909090909091, "Indonesian,English,Filipino,Malay": 0.23295454545454544, "Indonesian,English,Filipino,Vietnamese": 0.25, "Indonesian,English,Spanish,Chinese": 0.3068181818181818, "Indonesian,English,Spanish,Malay": 0.3352272727272727, "Indonesian,English,Spanish,Vietnamese": 0.3409090909090909, "Indonesian,English,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,English,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Chinese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Malay": 0.29545454545454547, "Indonesian,Filipino,Spanish,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Chinese,Malay": 0.23863636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Spanish,Chinese,Malay": 0.29545454545454547, "Indonesian,Spanish,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese": 0.23295454545454544, "English,Filipino,Spanish,Malay": 0.24431818181818182, "English,Filipino,Spanish,Vietnamese": 0.2727272727272727, "English,Filipino,Chinese,Malay": 0.2159090909090909, "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, "English,Filipino,Malay,Vietnamese": 0.2159090909090909, "English,Spanish,Chinese,Malay": 0.3125, "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, "English,Spanish,Malay,Vietnamese": 0.3181818181818182, "English,Chinese,Malay,Vietnamese": 0.2556818181818182, "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.2159090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,English,Filipino,Malay,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.25, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,English,Spanish,Malay,Vietnamese": 0.26704545454545453, "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "English,Filipino,Spanish,Chinese,Malay": 0.1875, "English,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "English,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1875 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 } }, "AC3_2": 0.47738614495027465, "AC3_3": 0.39184361489113734, "AC3_4": 0.3300928503238267, "AC3_5": 0.2835493293509241, "AC3_6": 0.2481447123896104, "AC3_7": 0.2204577151897091 }, "prompt_4": { "overall_acc": 0.4537337662337662, "language_acc": { "Indonesian": 0.4431818181818182, "English": 0.5681818181818182, "Filipino": 0.3409090909090909, "Spanish": 0.4659090909090909, "Chinese": 0.48863636363636365, "Malay": 0.4090909090909091, "Vietnamese": 0.4602272727272727 }, "consistency_score_2": 0.5454545454545454, "consistency_score_3": 0.37775974025974035, "consistency_score_4": 0.28847402597402594, "consistency_score_5": 0.23214285714285715, "consistency_score_6": 0.19318181818181818, "consistency_score_7": 0.16477272727272727, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5852272727272727, "Indonesian,Filipino": 0.5852272727272727, "Indonesian,Spanish": 0.6079545454545454, "Indonesian,Chinese": 0.5568181818181818, "Indonesian,Malay": 0.6136363636363636, "Indonesian,Vietnamese": 0.6306818181818182, "English,Filipino": 0.4431818181818182, "English,Spanish": 0.6420454545454546, "English,Chinese": 0.6022727272727273, "English,Malay": 0.5397727272727273, "English,Vietnamese": 0.5511363636363636, "Filipino,Spanish": 0.4715909090909091, "Filipino,Chinese": 0.45454545454545453, "Filipino,Malay": 0.45454545454545453, "Filipino,Vietnamese": 0.5, "Spanish,Chinese": 0.5511363636363636, "Spanish,Malay": 0.5681818181818182, "Spanish,Vietnamese": 0.5511363636363636, "Chinese,Malay": 0.4772727272727273, "Chinese,Vietnamese": 0.5113636363636364, "Malay,Vietnamese": 0.5568181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.3693181818181818, "Indonesian,English,Spanish": 0.4715909090909091, "Indonesian,English,Chinese": 0.4034090909090909, "Indonesian,English,Malay": 0.4090909090909091, "Indonesian,English,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Spanish": 0.3806818181818182, "Indonesian,Filipino,Chinese": 0.375, "Indonesian,Filipino,Malay": 0.3806818181818182, "Indonesian,Filipino,Vietnamese": 0.4090909090909091, "Indonesian,Spanish,Chinese": 0.4090909090909091, "Indonesian,Spanish,Malay": 0.44886363636363635, "Indonesian,Spanish,Vietnamese": 0.44886363636363635, "Indonesian,Chinese,Malay": 0.3806818181818182, "Indonesian,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,Malay,Vietnamese": 0.4431818181818182, "English,Filipino,Spanish": 0.3352272727272727, "English,Filipino,Chinese": 0.3125, "English,Filipino,Malay": 0.2897727272727273, "English,Filipino,Vietnamese": 0.32954545454545453, "English,Spanish,Chinese": 0.44886363636363635, "English,Spanish,Malay": 0.42045454545454547, "English,Spanish,Vietnamese": 0.4318181818181818, "English,Chinese,Malay": 0.3465909090909091, "English,Chinese,Vietnamese": 0.39204545454545453, "English,Malay,Vietnamese": 0.3806818181818182, "Filipino,Spanish,Chinese": 0.3125, "Filipino,Spanish,Malay": 0.3181818181818182, "Filipino,Spanish,Vietnamese": 0.32954545454545453, "Filipino,Chinese,Malay": 0.2784090909090909, "Filipino,Chinese,Vietnamese": 0.3125, "Filipino,Malay,Vietnamese": 0.3181818181818182, "Spanish,Chinese,Malay": 0.35795454545454547, "Spanish,Chinese,Vietnamese": 0.375, "Spanish,Malay,Vietnamese": 0.3977272727272727, "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.30113636363636365, "Indonesian,English,Filipino,Chinese": 0.2727272727272727, "Indonesian,English,Filipino,Malay": 0.26704545454545453, "Indonesian,English,Filipino,Vietnamese": 0.3068181818181818, "Indonesian,English,Spanish,Chinese": 0.3522727272727273, "Indonesian,English,Spanish,Malay": 0.3465909090909091, "Indonesian,English,Spanish,Vietnamese": 0.3806818181818182, "Indonesian,English,Chinese,Malay": 0.2840909090909091, "Indonesian,English,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.2840909090909091, "Indonesian,Filipino,Spanish,Malay": 0.29545454545454547, "Indonesian,Filipino,Spanish,Vietnamese": 0.29545454545454547, "Indonesian,Filipino,Chinese,Malay": 0.26704545454545453, "Indonesian,Filipino,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.30113636363636365, "Indonesian,Spanish,Chinese,Malay": 0.3125, "Indonesian,Spanish,Chinese,Vietnamese": 0.32954545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.3409090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.30113636363636365, "English,Filipino,Spanish,Chinese": 0.2556818181818182, "English,Filipino,Spanish,Malay": 0.23863636363636365, "English,Filipino,Spanish,Vietnamese": 0.2727272727272727, "English,Filipino,Chinese,Malay": 0.2159090909090909, "English,Filipino,Chinese,Vietnamese": 0.25, "English,Filipino,Malay,Vietnamese": 0.23295454545454544, "English,Spanish,Chinese,Malay": 0.30113636363636365, "English,Spanish,Chinese,Vietnamese": 0.32954545454545453, "English,Spanish,Malay,Vietnamese": 0.3125, "English,Chinese,Malay,Vietnamese": 0.26704545454545453, "Filipino,Spanish,Chinese,Malay": 0.23295454545454544, "Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.25, "Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.23863636363636365, "Indonesian,English,Filipino,Spanish,Malay": 0.22727272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.20454545454545456, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "English,Filipino,Spanish,Chinese,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1875, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 } }, "AC3_2": 0.4953843881049151, "AC3_3": 0.41227585860952487, "AC3_4": 0.352705556644256, "AC3_5": 0.3071428570980761, "AC3_6": 0.27098779509897997, "AC3_7": 0.2417531614975395 }, "prompt_5": { "overall_acc": 0.44642857142857145, "language_acc": { "Indonesian": 0.4318181818181818, "English": 0.5625, "Filipino": 0.3693181818181818, "Spanish": 0.4431818181818182, "Chinese": 0.45454545454545453, "Malay": 0.42045454545454547, "Vietnamese": 0.4431818181818182 }, "consistency_score_2": 0.5343614718614718, "consistency_score_3": 0.3616883116883116, "consistency_score_4": 0.26866883116883117, "consistency_score_5": 0.21076839826839824, "consistency_score_6": 0.17288961038961043, "consistency_score_7": 0.14772727272727273, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5454545454545454, "Indonesian,Filipino": 0.5227272727272727, "Indonesian,Spanish": 0.5852272727272727, "Indonesian,Chinese": 0.5056818181818182, "Indonesian,Malay": 0.5795454545454546, "Indonesian,Vietnamese": 0.5170454545454546, "English,Filipino": 0.4659090909090909, "English,Spanish": 0.6477272727272727, "English,Chinese": 0.6136363636363636, "English,Malay": 0.5284090909090909, "English,Vietnamese": 0.5, "Filipino,Spanish": 0.5284090909090909, "Filipino,Chinese": 0.4772727272727273, "Filipino,Malay": 0.4659090909090909, "Filipino,Vietnamese": 0.5056818181818182, "Spanish,Chinese": 0.6420454545454546, "Spanish,Malay": 0.5340909090909091, "Spanish,Vietnamese": 0.5397727272727273, "Chinese,Malay": 0.48863636363636365, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.5397727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.3409090909090909, "Indonesian,English,Spanish": 0.4431818181818182, "Indonesian,English,Chinese": 0.38636363636363635, "Indonesian,English,Malay": 0.39204545454545453, "Indonesian,English,Vietnamese": 0.36363636363636365, "Indonesian,Filipino,Spanish": 0.3693181818181818, "Indonesian,Filipino,Chinese": 0.3409090909090909, "Indonesian,Filipino,Malay": 0.3352272727272727, "Indonesian,Filipino,Vietnamese": 0.32386363636363635, "Indonesian,Spanish,Chinese": 0.42045454545454547, "Indonesian,Spanish,Malay": 0.4034090909090909, "Indonesian,Spanish,Vietnamese": 0.375, "Indonesian,Chinese,Malay": 0.32954545454545453, "Indonesian,Chinese,Vietnamese": 0.32954545454545453, "Indonesian,Malay,Vietnamese": 0.3693181818181818, "English,Filipino,Spanish": 0.3806818181818182, "English,Filipino,Chinese": 0.3409090909090909, "English,Filipino,Malay": 0.29545454545454547, "English,Filipino,Vietnamese": 0.3181818181818182, "English,Spanish,Chinese": 0.5, "English,Spanish,Malay": 0.3977272727272727, "English,Spanish,Vietnamese": 0.4034090909090909, "English,Chinese,Malay": 0.3465909090909091, "English,Chinese,Vietnamese": 0.375, "English,Malay,Vietnamese": 0.35795454545454547, "Filipino,Spanish,Chinese": 0.375, "Filipino,Spanish,Malay": 0.32386363636363635, "Filipino,Spanish,Vietnamese": 0.3522727272727273, "Filipino,Chinese,Malay": 0.2784090909090909, "Filipino,Chinese,Vietnamese": 0.3125, "Filipino,Malay,Vietnamese": 0.3181818181818182, "Spanish,Chinese,Malay": 0.375, "Spanish,Chinese,Vietnamese": 0.4090909090909091, "Spanish,Malay,Vietnamese": 0.36363636363636365, "Chinese,Malay,Vietnamese": 0.3125 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.29545454545454547, "Indonesian,English,Filipino,Chinese": 0.26704545454545453, "Indonesian,English,Filipino,Malay": 0.24431818181818182, "Indonesian,English,Filipino,Vietnamese": 0.24431818181818182, "Indonesian,English,Spanish,Chinese": 0.35795454545454547, "Indonesian,English,Spanish,Malay": 0.3181818181818182, "Indonesian,English,Spanish,Vietnamese": 0.30113636363636365, "Indonesian,English,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,English,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.2840909090909091, "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Chinese,Malay": 0.2159090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay": 0.2840909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.23863636363636365, "English,Filipino,Spanish,Chinese": 0.30113636363636365, "English,Filipino,Spanish,Malay": 0.25, "English,Filipino,Spanish,Vietnamese": 0.2840909090909091, "English,Filipino,Chinese,Malay": 0.22727272727272727, "English,Filipino,Chinese,Vietnamese": 0.2556818181818182, "English,Filipino,Malay,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese,Malay": 0.3125, "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, "English,Spanish,Malay,Vietnamese": 0.2840909090909091, "English,Chinese,Malay,Vietnamese": 0.2556818181818182, "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Filipino,Spanish,Malay,Vietnamese": 0.25, "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.24431818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2159090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.1875, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,English,Filipino,Malay,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.25, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "English,Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1875 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 } }, "AC3_2": 0.4864532019208453, "AC3_3": 0.3996154507913924, "AC3_4": 0.33545484023199346, "AC3_5": 0.28634652703988495, "AC3_6": 0.24925107653717393, "AC3_7": 0.22199453548176265 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6699029126213593 }, "prompt_2": { "accuracy": 0.6504854368932039 }, "prompt_3": { "accuracy": 0.6116504854368932 }, "prompt_4": { "accuracy": 0.6699029126213593 }, "prompt_5": { "accuracy": 0.6407766990291263 } }, "cn_eval": { "prompt_1": { "accuracy": 0.3333333333333333 }, "prompt_2": { "accuracy": 0.3333333333333333 }, "prompt_3": { "accuracy": 0.3619047619047619 }, "prompt_4": { "accuracy": 0.38095238095238093 }, "prompt_5": { "accuracy": 0.4 } }, "us_eval": { "prompt_1": { "accuracy": 0.6261682242990654 }, "prompt_2": { "accuracy": 0.5981308411214953 }, "prompt_3": { "accuracy": 0.5981308411214953 }, "prompt_4": { "accuracy": 0.6355140186915887 }, "prompt_5": { "accuracy": 0.5887850467289719 } }, "ph_eval": { "prompt_1": { "accuracy": 0.47, "category_acc": { "brand": 0.3, "demographics": 0.0, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.4, "politics": 0.7, "culture": 0.8, "film": 0.5, "law": 0.6, "geography": 0.6 } }, "prompt_2": { "accuracy": 0.49, "category_acc": { "brand": 0.5, "demographics": 0.4, "biology": 0.3, "history": 0.2, "literature": 0.3, "politics": 0.8, "culture": 0.9, "film": 0.5, "law": 0.5, "geography": 0.6 } }, "prompt_3": { "accuracy": 0.5, "category_acc": { "brand": 0.7, "demographics": 0.4, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.8, "culture": 0.7, "film": 0.3, "law": 0.5, "geography": 0.6 } }, "prompt_4": { "accuracy": 0.49, "category_acc": { "brand": 0.7, "demographics": 0.4, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.4, "politics": 0.7, "culture": 0.7, "film": 0.4, "law": 0.4, "geography": 0.5 } }, "prompt_5": { "accuracy": 0.48, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.3333333333333333, "literature": 0.5, "politics": 0.8, "culture": 0.8, "film": 0.4, "law": 0.3, "geography": 0.7 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.17850204847572934 }, "prompt_2": { "bleu_score": 0.1808454080159326 }, "prompt_3": { "bleu_score": 0.18662234818554896 }, "prompt_4": { "bleu_score": 0.176343510769412 }, "prompt_5": { "bleu_score": 0.1324532828678051 } }, "indommlu": { "prompt_1": { "accuracy": 0.42305894919554043, "category_acc": { "History": 0.39759036144578314, "Geography": 0.38571428571428573, "Lampungic": 0.29931972789115646, "Social science": 0.5809682804674458, "Balinese": 0.3205944798301486, "Makassarese": 0.3817204301075269, "Banjarese": 0.3611111111111111, "Chemistry": 0.3021897810218978, "Biology": 0.3940828402366864, "Science": 0.5201238390092879, "Christian religion": 0.48258706467661694, "Art": 0.46589018302828616, "Islam religion": 0.5049786628733998, "Hindu religion": 0.4866666666666667, "Madurese": 0.288135593220339, "Sport": 0.4391891891891892, "Indonesian language": 0.47665006226650064, "Physics": 0.35555555555555557, "Minangkabau culture": 0.3417085427135678, "Dayak language": 0.23853211009174313, "Sociology": 0.43548387096774194, "Economy": 0.41188524590163933, "Sundanese": 0.34831460674157305, "Javanese": 0.33669354838709675, "Civic education": 0.4721030042918455 } }, "prompt_2": { "accuracy": 0.4210561452700447, "category_acc": { "History": 0.41365461847389556, "Geography": 0.39387755102040817, "Lampungic": 0.2653061224489796, "Social science": 0.5876460767946577, "Balinese": 0.3078556263269639, "Makassarese": 0.3387096774193548, "Banjarese": 0.3611111111111111, "Chemistry": 0.2948905109489051, "Biology": 0.39289940828402364, "Science": 0.4984520123839009, "Christian religion": 0.48756218905472637, "Art": 0.4858569051580699, "Islam religion": 0.4950213371266003, "Hindu religion": 0.5, "Madurese": 0.31186440677966104, "Sport": 0.46621621621621623, "Indonesian language": 0.4747820672478207, "Physics": 0.3515151515151515, "Minangkabau culture": 0.31155778894472363, "Dayak language": 0.22018348623853212, "Sociology": 0.41330645161290325, "Economy": 0.4057377049180328, "Sundanese": 0.35350043215211757, "Javanese": 0.3387096774193548, "Civic education": 0.47639484978540775 } }, "prompt_3": { "accuracy": 0.4227251485412911, "category_acc": { "History": 0.42369477911646586, "Geography": 0.3816326530612245, "Lampungic": 0.2653061224489796, "Social science": 0.5876460767946577, "Balinese": 0.37154989384288745, "Makassarese": 0.3763440860215054, "Banjarese": 0.3541666666666667, "Chemistry": 0.28175182481751826, "Biology": 0.36923076923076925, "Science": 0.49019607843137253, "Christian religion": 0.48756218905472637, "Art": 0.46422628951747086, "Islam religion": 0.5021337126600285, "Hindu religion": 0.5066666666666667, "Madurese": 0.3288135593220339, "Sport": 0.4594594594594595, "Indonesian language": 0.47322540473225405, "Physics": 0.3797979797979798, "Minangkabau culture": 0.2864321608040201, "Dayak language": 0.3119266055045872, "Sociology": 0.43548387096774194, "Economy": 0.4016393442622951, "Sundanese": 0.34485738980121, "Javanese": 0.3467741935483871, "Civic education": 0.4892703862660944 } }, "prompt_4": { "accuracy": 0.4135790106148608, "category_acc": { "History": 0.40763052208835343, "Geography": 0.3877551020408163, "Lampungic": 0.2585034013605442, "Social science": 0.5626043405676127, "Balinese": 0.33970276008492567, "Makassarese": 0.34408602150537637, "Banjarese": 0.3402777777777778, "Chemistry": 0.29781021897810217, "Biology": 0.37514792899408284, "Science": 0.5025799793601651, "Christian religion": 0.48258706467661694, "Art": 0.47420965058236275, "Islam religion": 0.4907539118065434, "Hindu religion": 0.4866666666666667, "Madurese": 0.2983050847457627, "Sport": 0.4797297297297297, "Indonesian language": 0.462640099626401, "Physics": 0.3515151515151515, "Minangkabau culture": 0.32160804020100503, "Dayak language": 0.25688073394495414, "Sociology": 0.4274193548387097, "Economy": 0.38729508196721313, "Sundanese": 0.34312878133102853, "Javanese": 0.31149193548387094, "Civic education": 0.46924177396280403 } }, "prompt_5": { "accuracy": 0.4085052406702717, "category_acc": { "History": 0.41566265060240964, "Geography": 0.38571428571428573, "Lampungic": 0.2925170068027211, "Social science": 0.5809682804674458, "Balinese": 0.34394904458598724, "Makassarese": 0.3548387096774194, "Banjarese": 0.3958333333333333, "Chemistry": 0.2656934306569343, "Biology": 0.37514792899408284, "Science": 0.48297213622291024, "Christian religion": 0.4527363184079602, "Art": 0.46755407653910147, "Islam religion": 0.47368421052631576, "Hindu religion": 0.46, "Madurese": 0.3016949152542373, "Sport": 0.4391891891891892, "Indonesian language": 0.4576587795765878, "Physics": 0.33131313131313134, "Minangkabau culture": 0.2914572864321608, "Dayak language": 0.24770642201834864, "Sociology": 0.4254032258064516, "Economy": 0.3770491803278688, "Sundanese": 0.3292999135695765, "Javanese": 0.3316532258064516, "Civic education": 0.46924177396280403 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.29083824388253576 }, "prompt_2": { "bleu_score": 0.30095965158253596 }, "prompt_3": { "bleu_score": 0.2998787922647266 }, "prompt_4": { "bleu_score": 0.2990516384145601 }, "prompt_5": { "bleu_score": 0.282716600632688 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.21256255595737264 }, "prompt_2": { "bleu_score": 0.22028609039927588 }, "prompt_3": { "bleu_score": 0.21794217050063686 }, "prompt_4": { "bleu_score": 0.21639144518942477 }, "prompt_5": { "bleu_score": 0.2165329134969405 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.1772473628232291 }, "prompt_2": { "bleu_score": 0.1790645913924739 }, "prompt_3": { "bleu_score": 0.17880842836525265 }, "prompt_4": { "bleu_score": 0.18272545312220376 }, "prompt_5": { "bleu_score": 0.17708566347520507 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.28560213975589965 }, "prompt_2": { "bleu_score": 0.2917520756266462 }, "prompt_3": { "bleu_score": 0.2875298051907796 }, "prompt_4": { "bleu_score": 0.2897057158686528 }, "prompt_5": { "bleu_score": 0.2792743718264202 } }, "mmlu": { "prompt_1": { "accuracy": 0.5787631271878646 }, "prompt_2": { "accuracy": 0.572928821470245 }, "prompt_3": { "accuracy": 0.5927654609101517 }, "prompt_4": { "accuracy": 0.5869311551925321 }, "prompt_5": { "accuracy": 0.5799299883313885 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.5526635681086879, "category_acc": { "high_school_european_history": 0.6829268292682927, "business_ethics": 0.6060606060606061, "clinical_knowledge": 0.6439393939393939, "medical_genetics": 0.6666666666666666, "high_school_us_history": 0.7192118226600985, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.711864406779661, "virology": 0.47878787878787876, "high_school_microeconomics": 0.5780590717299579, "econometrics": 0.3805309734513274, "college_computer_science": 0.41414141414141414, "high_school_biology": 0.7055016181229773, "abstract_algebra": 0.26262626262626265, "professional_accounting": 0.40213523131672596, "philosophy": 0.6064516129032258, "professional_medicine": 0.6125461254612546, "nutrition": 0.5770491803278689, "global_facts": 0.40404040404040403, "machine_learning": 0.3963963963963964, "security_studies": 0.5819672131147541, "public_relations": 0.5504587155963303, "professional_psychology": 0.5564648117839607, "prehistory": 0.6253869969040248, "anatomy": 0.5, "human_sexuality": 0.6153846153846154, "college_medicine": 0.5755813953488372, "high_school_government_and_politics": 0.8125, "college_chemistry": 0.42424242424242425, "logical_fallacies": 0.654320987654321, "high_school_geography": 0.6700507614213198, "elementary_mathematics": 0.46684350132625996, "human_aging": 0.6081081081081081, "college_mathematics": 0.24242424242424243, "high_school_psychology": 0.7665441176470589, "formal_logic": 0.368, "high_school_statistics": 0.46511627906976744, "international_law": 0.7, "high_school_mathematics": 0.26394052044609667, "high_school_computer_science": 0.6363636363636364, "conceptual_physics": 0.47863247863247865, "miscellaneous": 0.7672634271099744, "high_school_chemistry": 0.400990099009901, "marketing": 0.8454935622317596, "professional_law": 0.41030658838878015, "management": 0.7745098039215687, "college_physics": 0.39603960396039606, "jurisprudence": 0.6448598130841121, "world_religions": 0.7823529411764706, "sociology": 0.765, "us_foreign_policy": 0.7575757575757576, "high_school_macroeconomics": 0.4884318766066838, "computer_security": 0.6464646464646465, "moral_scenarios": 0.29977628635346754, "moral_disputes": 0.591304347826087, "electrical_engineering": 0.5208333333333334, "astronomy": 0.5960264900662252, "college_biology": 0.6433566433566433 } }, "prompt_2": { "accuracy": 0.5636753664640687, "category_acc": { "high_school_european_history": 0.7073170731707317, "business_ethics": 0.6161616161616161, "clinical_knowledge": 0.6590909090909091, "medical_genetics": 0.6363636363636364, "high_school_us_history": 0.7586206896551724, "high_school_physics": 0.37333333333333335, "high_school_world_history": 0.7372881355932204, "virology": 0.48484848484848486, "high_school_microeconomics": 0.6033755274261603, "econometrics": 0.3805309734513274, "college_computer_science": 0.41414141414141414, "high_school_biology": 0.6699029126213593, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.4483985765124555, "philosophy": 0.603225806451613, "professional_medicine": 0.6162361623616236, "nutrition": 0.5836065573770491, "global_facts": 0.40404040404040403, "machine_learning": 0.3963963963963964, "security_studies": 0.6188524590163934, "public_relations": 0.5963302752293578, "professional_psychology": 0.5695581014729951, "prehistory": 0.6377708978328174, "anatomy": 0.5223880597014925, "human_sexuality": 0.5923076923076923, "college_medicine": 0.5755813953488372, "high_school_government_and_politics": 0.8229166666666666, "college_chemistry": 0.41414141414141414, "logical_fallacies": 0.6790123456790124, "high_school_geography": 0.6649746192893401, "elementary_mathematics": 0.44562334217506633, "human_aging": 0.6576576576576577, "college_mathematics": 0.32323232323232326, "high_school_psychology": 0.7628676470588235, "formal_logic": 0.4, "high_school_statistics": 0.4744186046511628, "international_law": 0.725, "high_school_mathematics": 0.2788104089219331, "high_school_computer_science": 0.6262626262626263, "conceptual_physics": 0.5213675213675214, "miscellaneous": 0.7774936061381074, "high_school_chemistry": 0.4405940594059406, "marketing": 0.8540772532188842, "professional_law": 0.41030658838878015, "management": 0.7352941176470589, "college_physics": 0.44554455445544555, "jurisprudence": 0.6728971962616822, "world_religions": 0.788235294117647, "sociology": 0.74, "us_foreign_policy": 0.7373737373737373, "high_school_macroeconomics": 0.4961439588688946, "computer_security": 0.6767676767676768, "moral_scenarios": 0.31208053691275167, "moral_disputes": 0.6144927536231884, "electrical_engineering": 0.5208333333333334, "astronomy": 0.6357615894039735, "college_biology": 0.6503496503496503 } }, "prompt_3": { "accuracy": 0.5614587057561673, "category_acc": { "high_school_european_history": 0.7134146341463414, "business_ethics": 0.6262626262626263, "clinical_knowledge": 0.6477272727272727, "medical_genetics": 0.6666666666666666, "high_school_us_history": 0.7635467980295566, "high_school_physics": 0.35333333333333333, "high_school_world_history": 0.7542372881355932, "virology": 0.48484848484848486, "high_school_microeconomics": 0.5991561181434599, "econometrics": 0.35398230088495575, "college_computer_science": 0.41414141414141414, "high_school_biology": 0.6601941747572816, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.43416370106761565, "philosophy": 0.6129032258064516, "professional_medicine": 0.5830258302583026, "nutrition": 0.6, "global_facts": 0.37373737373737376, "machine_learning": 0.3963963963963964, "security_studies": 0.6475409836065574, "public_relations": 0.6238532110091743, "professional_psychology": 0.5695581014729951, "prehistory": 0.6346749226006192, "anatomy": 0.5671641791044776, "human_sexuality": 0.5923076923076923, "college_medicine": 0.5581395348837209, "high_school_government_and_politics": 0.7864583333333334, "college_chemistry": 0.3939393939393939, "logical_fallacies": 0.654320987654321, "high_school_geography": 0.6700507614213198, "elementary_mathematics": 0.44562334217506633, "human_aging": 0.6486486486486487, "college_mathematics": 0.3333333333333333, "high_school_psychology": 0.7738970588235294, "formal_logic": 0.424, "high_school_statistics": 0.4511627906976744, "international_law": 0.7, "high_school_mathematics": 0.26394052044609667, "high_school_computer_science": 0.6565656565656566, "conceptual_physics": 0.5085470085470085, "miscellaneous": 0.7800511508951407, "high_school_chemistry": 0.4504950495049505, "marketing": 0.8283261802575107, "professional_law": 0.4024787997390737, "management": 0.7352941176470589, "college_physics": 0.40594059405940597, "jurisprudence": 0.616822429906542, "world_religions": 0.8117647058823529, "sociology": 0.79, "us_foreign_policy": 0.7777777777777778, "high_school_macroeconomics": 0.4987146529562982, "computer_security": 0.6262626262626263, "moral_scenarios": 0.28859060402684567, "moral_disputes": 0.6521739130434783, "electrical_engineering": 0.5277777777777778, "astronomy": 0.6158940397350994, "college_biology": 0.6713286713286714 } }, "prompt_4": { "accuracy": 0.5591705398641401, "category_acc": { "high_school_european_history": 0.6829268292682927, "business_ethics": 0.6060606060606061, "clinical_knowledge": 0.6590909090909091, "medical_genetics": 0.6464646464646465, "high_school_us_history": 0.7389162561576355, "high_school_physics": 0.34, "high_school_world_history": 0.7415254237288136, "virology": 0.45454545454545453, "high_school_microeconomics": 0.6244725738396625, "econometrics": 0.3805309734513274, "college_computer_science": 0.3838383838383838, "high_school_biology": 0.686084142394822, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.4199288256227758, "philosophy": 0.6290322580645161, "professional_medicine": 0.5830258302583026, "nutrition": 0.580327868852459, "global_facts": 0.35353535353535354, "machine_learning": 0.36936936936936937, "security_studies": 0.6229508196721312, "public_relations": 0.5779816513761468, "professional_psychology": 0.5662847790507365, "prehistory": 0.6439628482972136, "anatomy": 0.582089552238806, "human_sexuality": 0.6230769230769231, "college_medicine": 0.5755813953488372, "high_school_government_and_politics": 0.7864583333333334, "college_chemistry": 0.3939393939393939, "logical_fallacies": 0.691358024691358, "high_school_geography": 0.7106598984771574, "elementary_mathematics": 0.4217506631299735, "human_aging": 0.6351351351351351, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.7702205882352942, "formal_logic": 0.376, "high_school_statistics": 0.44651162790697674, "international_law": 0.725, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.6363636363636364, "conceptual_physics": 0.5213675213675214, "miscellaneous": 0.7672634271099744, "high_school_chemistry": 0.44554455445544555, "marketing": 0.8497854077253219, "professional_law": 0.41030658838878015, "management": 0.7352941176470589, "college_physics": 0.3465346534653465, "jurisprudence": 0.6542056074766355, "world_religions": 0.8117647058823529, "sociology": 0.76, "us_foreign_policy": 0.7575757575757576, "high_school_macroeconomics": 0.46786632390745503, "computer_security": 0.7070707070707071, "moral_scenarios": 0.29082774049217003, "moral_disputes": 0.6376811594202898, "electrical_engineering": 0.5486111111111112, "astronomy": 0.6158940397350994, "college_biology": 0.6713286713286714 } }, "prompt_5": { "accuracy": 0.5610296746514122, "category_acc": { "high_school_european_history": 0.7012195121951219, "business_ethics": 0.6060606060606061, "clinical_knowledge": 0.6212121212121212, "medical_genetics": 0.6565656565656566, "high_school_us_history": 0.7487684729064039, "high_school_physics": 0.3466666666666667, "high_school_world_history": 0.7330508474576272, "virology": 0.4666666666666667, "high_school_microeconomics": 0.5864978902953587, "econometrics": 0.40707964601769914, "college_computer_science": 0.42424242424242425, "high_school_biology": 0.6763754045307443, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.43416370106761565, "philosophy": 0.6258064516129033, "professional_medicine": 0.5793357933579336, "nutrition": 0.5672131147540984, "global_facts": 0.36363636363636365, "machine_learning": 0.38738738738738737, "security_studies": 0.5983606557377049, "public_relations": 0.6238532110091743, "professional_psychology": 0.5646481178396072, "prehistory": 0.653250773993808, "anatomy": 0.5223880597014925, "human_sexuality": 0.6461538461538462, "college_medicine": 0.5872093023255814, "high_school_government_and_politics": 0.7916666666666666, "college_chemistry": 0.37373737373737376, "logical_fallacies": 0.7283950617283951, "high_school_geography": 0.6751269035532995, "elementary_mathematics": 0.4350132625994695, "human_aging": 0.6306306306306306, "college_mathematics": 0.23232323232323232, "high_school_psychology": 0.7702205882352942, "formal_logic": 0.416, "high_school_statistics": 0.5162790697674419, "international_law": 0.7, "high_school_mathematics": 0.29739776951672864, "high_school_computer_science": 0.6767676767676768, "conceptual_physics": 0.5170940170940171, "miscellaneous": 0.7634271099744245, "high_school_chemistry": 0.44554455445544555, "marketing": 0.8369098712446352, "professional_law": 0.410958904109589, "management": 0.7450980392156863, "college_physics": 0.37623762376237624, "jurisprudence": 0.6728971962616822, "world_religions": 0.7823529411764706, "sociology": 0.755, "us_foreign_policy": 0.7777777777777778, "high_school_macroeconomics": 0.5115681233933161, "computer_security": 0.6666666666666666, "moral_scenarios": 0.29194630872483224, "moral_disputes": 0.6463768115942029, "electrical_engineering": 0.5208333333333334, "astronomy": 0.6225165562913907, "college_biology": 0.6573426573426573 } } }, "c_eval": { "prompt_1": { "accuracy": 0.39821693907875183 }, "prompt_2": { "accuracy": 0.4049034175334324 }, "prompt_3": { "accuracy": 0.41233283803863297 }, "prompt_4": { "accuracy": 0.38558692421991086 }, "prompt_5": { "accuracy": 0.40936106983655274 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.4078455790784558, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.4583333333333333, "computer_architecture": 0.5, "college_programming": 0.5714285714285714, "college_physics": 0.25, "college_chemistry": 0.1724137931034483, "advanced_mathematics": 0.16666666666666666, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.4782608695652174, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.25, "middle_school_biology": 0.6153846153846154, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.36666666666666664, "business_administration": 0.2894736842105263, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.4827586206896552, "education_science": 0.5882352941176471, "teacher_qualification": 0.46938775510204084, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.5384615384615384, "middle_school_geography": 0.17647058823529413, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.48148148148148145, "law": 0.41379310344827586, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.21052631578947367, "professional_tour_guide": 0.38235294117647056, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.56, "middle_school_history": 0.5925925925925926, "civil_servant": 0.40384615384615385, "sports_science": 0.4166666666666667, "plant_protection": 0.4444444444444444, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.49019607843137253, "accountant": 0.4074074074074074, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.4444444444444444, "tax_accountant": 0.4444444444444444, "physician": 0.3148148148148148 } }, "prompt_2": { "accuracy": 0.3991282689912827, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5833333333333334, "computer_architecture": 0.38461538461538464, "college_programming": 0.5476190476190477, "college_physics": 0.20833333333333334, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.25, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.42857142857142855, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3103448275862069, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.375, "middle_school_mathematics": 0.5, "middle_school_biology": 0.5, "middle_school_physics": 0.5416666666666666, "middle_school_chemistry": 0.56, "veterinary_medicine": 0.42857142857142855, "college_economics": 0.36666666666666664, "business_administration": 0.2894736842105263, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.4117647058823529, "teacher_qualification": 0.5102040816326531, "high_school_politics": 0.5, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.5769230769230769, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.35714285714285715, "ideological_and_moral_cultivation": 0.625, "logic": 0.4444444444444444, "law": 0.41379310344827586, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.2631578947368421, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.25, "high_school_history": 0.52, "middle_school_history": 0.48148148148148145, "civil_servant": 0.4230769230769231, "sports_science": 0.375, "plant_protection": 0.4444444444444444, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.49019607843137253, "accountant": 0.46296296296296297, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.24074074074074073, "physician": 0.2962962962962963 } }, "prompt_3": { "accuracy": 0.4041095890410959, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.5, "computer_architecture": 0.5, "college_programming": 0.5238095238095238, "college_physics": 0.25, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.4166666666666667, "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.5384615384615384, "middle_school_physics": 0.5, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.38333333333333336, "business_administration": 0.3157894736842105, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.5588235294117647, "teacher_qualification": 0.5510204081632653, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.5769230769230769, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.35714285714285715, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.4444444444444444, "law": 0.3793103448275862, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.21052631578947367, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.52, "middle_school_history": 0.48148148148148145, "civil_servant": 0.4423076923076923, "sports_science": 0.4166666666666667, "plant_protection": 0.4444444444444444, "basic_medicine": 0.375, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.47058823529411764, "accountant": 0.5, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.4166666666666667, "tax_accountant": 0.25925925925925924, "physician": 0.2962962962962963 } }, "prompt_4": { "accuracy": 0.3941469489414695, "category_acc": { "computer_network": 0.4166666666666667, "operating_system": 0.4166666666666667, "computer_architecture": 0.5384615384615384, "college_programming": 0.5476190476190477, "college_physics": 0.25, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3103448275862069, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.25, "high_school_biology": 0.375, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.5, "middle_school_physics": 0.5, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.35, "business_administration": 0.3157894736842105, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.5588235294117647, "teacher_qualification": 0.5306122448979592, "high_school_politics": 0.625, "high_school_geography": 0.4166666666666667, "middle_school_politics": 0.5384615384615384, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.7083333333333334, "logic": 0.4444444444444444, "law": 0.3448275862068966, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.3157894736842105, "professional_tour_guide": 0.4117647058823529, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.6, "middle_school_history": 0.4074074074074074, "civil_servant": 0.40384615384615385, "sports_science": 0.375, "plant_protection": 0.4074074074074074, "basic_medicine": 0.375, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.5294117647058824, "accountant": 0.3333333333333333, "fire_engineer": 0.4166666666666667, "environmental_impact_assessment_engineer": 0.3888888888888889, "tax_accountant": 0.37037037037037035, "physician": 0.3333333333333333 } }, "prompt_5": { "accuracy": 0.40224159402241594, "category_acc": { "computer_network": 0.5, "operating_system": 0.5, "computer_architecture": 0.46153846153846156, "college_programming": 0.5476190476190477, "college_physics": 0.3333333333333333, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.25, "middle_school_biology": 0.6153846153846154, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.35, "business_administration": 0.3157894736842105, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.5862068965517241, "education_science": 0.5, "teacher_qualification": 0.5714285714285714, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.5416666666666666, "middle_school_politics": 0.5, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.39285714285714285, "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.4444444444444444, "law": 0.41379310344827586, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.23684210526315788, "professional_tour_guide": 0.38235294117647056, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.6, "middle_school_history": 0.6296296296296297, "civil_servant": 0.4423076923076923, "sports_science": 0.375, "plant_protection": 0.5185185185185185, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.45098039215686275, "accountant": 0.3888888888888889, "fire_engineer": 0.3333333333333333, "environmental_impact_assessment_engineer": 0.5277777777777778, "tax_accountant": 0.25925925925925924, "physician": 0.3148148148148148 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.37992831541218636 }, "prompt_2": { "accuracy": 0.34408602150537637 }, "prompt_3": { "accuracy": 0.34767025089605735 }, "prompt_4": { "accuracy": 0.36917562724014336 }, "prompt_5": { "accuracy": 0.3655913978494624 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.41037817302711105, "category_acc": { "agronomy": 0.39644970414201186, "anatomy": 0.27702702702702703, "ancient_chinese": 0.2804878048780488, "arts": 0.38125, "astronomy": 0.34545454545454546, "business_ethics": 0.48325358851674644, "chinese_civil_service_exam": 0.3875, "chinese_driving_rule": 0.5190839694656488, "chinese_food_culture": 0.3014705882352941, "chinese_foreign_policy": 0.4672897196261682, "chinese_history": 0.4582043343653251, "chinese_literature": 0.3333333333333333, "chinese_teacher_qualification": 0.5027932960893855, "clinical_knowledge": 0.34177215189873417, "college_actuarial_science": 0.25471698113207547, "college_education": 0.4766355140186916, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.25925925925925924, "college_mathematics": 0.23809523809523808, "college_medical_statistics": 0.41509433962264153, "college_medicine": 0.3626373626373626, "computer_science": 0.5049019607843137, "computer_security": 0.5789473684210527, "conceptual_physics": 0.41496598639455784, "construction_project_management": 0.39568345323741005, "economics": 0.46540880503144655, "education": 0.43558282208588955, "electrical_engineering": 0.436046511627907, "elementary_chinese": 0.24603174603174602, "elementary_commonsense": 0.41919191919191917, "elementary_information_and_technology": 0.6176470588235294, "elementary_mathematics": 0.3173913043478261, "ethnology": 0.34074074074074073, "food_science": 0.42657342657342656, "genetics": 0.3977272727272727, "global_facts": 0.3959731543624161, "high_school_biology": 0.3609467455621302, "high_school_chemistry": 0.3106060606060606, "high_school_geography": 0.423728813559322, "high_school_mathematics": 0.21951219512195122, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.4405594405594406, "human_sexuality": 0.4603174603174603, "international_law": 0.3675675675675676, "journalism": 0.436046511627907, "jurisprudence": 0.45742092457420924, "legal_and_moral_basis": 0.677570093457944, "logical": 0.42276422764227645, "machine_learning": 0.45081967213114754, "management": 0.49523809523809526, "marketing": 0.55, "marxist_theory": 0.43386243386243384, "modern_chinese": 0.31896551724137934, "nutrition": 0.4068965517241379, "philosophy": 0.4, "professional_accounting": 0.45714285714285713, "professional_law": 0.3459715639810427, "professional_medicine": 0.30851063829787234, "professional_psychology": 0.4267241379310345, "public_relations": 0.4942528735632184, "security_study": 0.42962962962962964, "sociology": 0.4690265486725664, "sports_science": 0.46060606060606063, "traditional_chinese_medicine": 0.3081081081081081, "virology": 0.42011834319526625, "world_history": 0.5093167701863354, "world_religions": 0.43125 } }, "prompt_2": { "accuracy": 0.40614747021239855, "category_acc": { "agronomy": 0.3905325443786982, "anatomy": 0.24324324324324326, "ancient_chinese": 0.27439024390243905, "arts": 0.375, "astronomy": 0.36363636363636365, "business_ethics": 0.5215311004784688, "chinese_civil_service_exam": 0.3875, "chinese_driving_rule": 0.5038167938931297, "chinese_food_culture": 0.2867647058823529, "chinese_foreign_policy": 0.5233644859813084, "chinese_history": 0.4520123839009288, "chinese_literature": 0.3284313725490196, "chinese_teacher_qualification": 0.48044692737430167, "clinical_knowledge": 0.31645569620253167, "college_actuarial_science": 0.25471698113207547, "college_education": 0.3925233644859813, "college_engineering_hydrology": 0.33962264150943394, "college_law": 0.25925925925925924, "college_mathematics": 0.24761904761904763, "college_medical_statistics": 0.37735849056603776, "college_medicine": 0.38461538461538464, "computer_science": 0.4950980392156863, "computer_security": 0.5789473684210527, "conceptual_physics": 0.41496598639455784, "construction_project_management": 0.4244604316546763, "economics": 0.4591194968553459, "education": 0.4233128834355828, "electrical_engineering": 0.4186046511627907, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.3787878787878788, "elementary_information_and_technology": 0.6176470588235294, "elementary_mathematics": 0.30869565217391304, "ethnology": 0.35555555555555557, "food_science": 0.42657342657342656, "genetics": 0.3806818181818182, "global_facts": 0.40939597315436244, "high_school_biology": 0.40828402366863903, "high_school_chemistry": 0.3333333333333333, "high_school_geography": 0.3898305084745763, "high_school_mathematics": 0.2926829268292683, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.43356643356643354, "human_sexuality": 0.48412698412698413, "international_law": 0.34054054054054056, "journalism": 0.436046511627907, "jurisprudence": 0.43309002433090027, "legal_and_moral_basis": 0.6542056074766355, "logical": 0.43902439024390244, "machine_learning": 0.4016393442622951, "management": 0.5, "marketing": 0.5555555555555556, "marxist_theory": 0.42328042328042326, "modern_chinese": 0.23275862068965517, "nutrition": 0.41379310344827586, "philosophy": 0.4, "professional_accounting": 0.4514285714285714, "professional_law": 0.36018957345971564, "professional_medicine": 0.300531914893617, "professional_psychology": 0.44396551724137934, "public_relations": 0.47701149425287354, "security_study": 0.4, "sociology": 0.4557522123893805, "sports_science": 0.4666666666666667, "traditional_chinese_medicine": 0.3081081081081081, "virology": 0.42011834319526625, "world_history": 0.45962732919254656, "world_religions": 0.3875 } }, "prompt_3": { "accuracy": 0.4114142635123467, "category_acc": { "agronomy": 0.40236686390532544, "anatomy": 0.2972972972972973, "ancient_chinese": 0.2682926829268293, "arts": 0.3875, "astronomy": 0.38181818181818183, "business_ethics": 0.49282296650717705, "chinese_civil_service_exam": 0.39375, "chinese_driving_rule": 0.5648854961832062, "chinese_food_culture": 0.29411764705882354, "chinese_foreign_policy": 0.4485981308411215, "chinese_history": 0.44891640866873067, "chinese_literature": 0.3284313725490196, "chinese_teacher_qualification": 0.49162011173184356, "clinical_knowledge": 0.34177215189873417, "college_actuarial_science": 0.32075471698113206, "college_education": 0.45794392523364486, "college_engineering_hydrology": 0.3490566037735849, "college_law": 0.26851851851851855, "college_mathematics": 0.23809523809523808, "college_medical_statistics": 0.3867924528301887, "college_medicine": 0.3626373626373626, "computer_science": 0.47549019607843135, "computer_security": 0.5555555555555556, "conceptual_physics": 0.4557823129251701, "construction_project_management": 0.43884892086330934, "economics": 0.4591194968553459, "education": 0.4233128834355828, "electrical_engineering": 0.4418604651162791, "elementary_chinese": 0.25396825396825395, "elementary_commonsense": 0.3939393939393939, "elementary_information_and_technology": 0.6554621848739496, "elementary_mathematics": 0.28695652173913044, "ethnology": 0.34814814814814815, "food_science": 0.4195804195804196, "genetics": 0.39204545454545453, "global_facts": 0.42953020134228187, "high_school_biology": 0.378698224852071, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.3728813559322034, "high_school_mathematics": 0.27439024390243905, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.42657342657342656, "human_sexuality": 0.46825396825396826, "international_law": 0.3675675675675676, "journalism": 0.4476744186046512, "jurisprudence": 0.45255474452554745, "legal_and_moral_basis": 0.6448598130841121, "logical": 0.4715447154471545, "machine_learning": 0.4262295081967213, "management": 0.49523809523809526, "marketing": 0.5666666666666667, "marxist_theory": 0.42328042328042326, "modern_chinese": 0.3017241379310345, "nutrition": 0.4413793103448276, "philosophy": 0.3904761904761905, "professional_accounting": 0.4514285714285714, "professional_law": 0.36492890995260663, "professional_medicine": 0.300531914893617, "professional_psychology": 0.41810344827586204, "public_relations": 0.47701149425287354, "security_study": 0.43703703703703706, "sociology": 0.4823008849557522, "sports_science": 0.4727272727272727, "traditional_chinese_medicine": 0.34054054054054056, "virology": 0.44970414201183434, "world_history": 0.43478260869565216, "world_religions": 0.40625 } }, "prompt_4": { "accuracy": 0.40312553963046105, "category_acc": { "agronomy": 0.3727810650887574, "anatomy": 0.25675675675675674, "ancient_chinese": 0.25609756097560976, "arts": 0.38125, "astronomy": 0.3393939393939394, "business_ethics": 0.5263157894736842, "chinese_civil_service_exam": 0.3625, "chinese_driving_rule": 0.5572519083969466, "chinese_food_culture": 0.3088235294117647, "chinese_foreign_policy": 0.5046728971962616, "chinese_history": 0.4520123839009288, "chinese_literature": 0.3235294117647059, "chinese_teacher_qualification": 0.5195530726256983, "clinical_knowledge": 0.35864978902953587, "college_actuarial_science": 0.3018867924528302, "college_education": 0.42990654205607476, "college_engineering_hydrology": 0.3584905660377358, "college_law": 0.2777777777777778, "college_mathematics": 0.20952380952380953, "college_medical_statistics": 0.3113207547169811, "college_medicine": 0.37362637362637363, "computer_science": 0.46568627450980393, "computer_security": 0.5672514619883041, "conceptual_physics": 0.40816326530612246, "construction_project_management": 0.4316546762589928, "economics": 0.4528301886792453, "education": 0.4723926380368098, "electrical_engineering": 0.42441860465116277, "elementary_chinese": 0.2619047619047619, "elementary_commonsense": 0.41414141414141414, "elementary_information_and_technology": 0.634453781512605, "elementary_mathematics": 0.29130434782608694, "ethnology": 0.37037037037037035, "food_science": 0.3916083916083916, "genetics": 0.375, "global_facts": 0.40939597315436244, "high_school_biology": 0.3668639053254438, "high_school_chemistry": 0.3106060606060606, "high_school_geography": 0.3898305084745763, "high_school_mathematics": 0.23170731707317074, "high_school_physics": 0.3181818181818182, "high_school_politics": 0.3706293706293706, "human_sexuality": 0.5, "international_law": 0.35135135135135137, "journalism": 0.45930232558139533, "jurisprudence": 0.45255474452554745, "legal_and_moral_basis": 0.6542056074766355, "logical": 0.4065040650406504, "machine_learning": 0.39344262295081966, "management": 0.47619047619047616, "marketing": 0.5444444444444444, "marxist_theory": 0.43915343915343913, "modern_chinese": 0.29310344827586204, "nutrition": 0.4, "philosophy": 0.4, "professional_accounting": 0.4114285714285714, "professional_law": 0.3459715639810427, "professional_medicine": 0.31117021276595747, "professional_psychology": 0.40948275862068967, "public_relations": 0.5057471264367817, "security_study": 0.362962962962963, "sociology": 0.4424778761061947, "sports_science": 0.37575757575757573, "traditional_chinese_medicine": 0.2648648648648649, "virology": 0.40236686390532544, "world_history": 0.5031055900621118, "world_religions": 0.44375 } }, "prompt_5": { "accuracy": 0.40493869797962356, "category_acc": { "agronomy": 0.3668639053254438, "anatomy": 0.32432432432432434, "ancient_chinese": 0.25, "arts": 0.3375, "astronomy": 0.38181818181818183, "business_ethics": 0.5358851674641149, "chinese_civil_service_exam": 0.40625, "chinese_driving_rule": 0.5419847328244275, "chinese_food_culture": 0.3161764705882353, "chinese_foreign_policy": 0.48598130841121495, "chinese_history": 0.4551083591331269, "chinese_literature": 0.3333333333333333, "chinese_teacher_qualification": 0.48044692737430167, "clinical_knowledge": 0.3291139240506329, "college_actuarial_science": 0.2358490566037736, "college_education": 0.4766355140186916, "college_engineering_hydrology": 0.37735849056603776, "college_law": 0.3148148148148148, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.4339622641509434, "college_medicine": 0.3333333333333333, "computer_science": 0.49019607843137253, "computer_security": 0.6023391812865497, "conceptual_physics": 0.3469387755102041, "construction_project_management": 0.43884892086330934, "economics": 0.4591194968553459, "education": 0.4294478527607362, "electrical_engineering": 0.42441860465116277, "elementary_chinese": 0.2777777777777778, "elementary_commonsense": 0.3888888888888889, "elementary_information_and_technology": 0.6134453781512605, "elementary_mathematics": 0.32608695652173914, "ethnology": 0.35555555555555557, "food_science": 0.34965034965034963, "genetics": 0.3465909090909091, "global_facts": 0.3959731543624161, "high_school_biology": 0.3609467455621302, "high_school_chemistry": 0.24242424242424243, "high_school_geography": 0.3983050847457627, "high_school_mathematics": 0.24390243902439024, "high_school_physics": 0.3, "high_school_politics": 0.4405594405594406, "human_sexuality": 0.4523809523809524, "international_law": 0.3675675675675676, "journalism": 0.42441860465116277, "jurisprudence": 0.45255474452554745, "legal_and_moral_basis": 0.719626168224299, "logical": 0.36585365853658536, "machine_learning": 0.4344262295081967, "management": 0.5, "marketing": 0.5666666666666667, "marxist_theory": 0.43386243386243384, "modern_chinese": 0.25862068965517243, "nutrition": 0.43448275862068964, "philosophy": 0.42857142857142855, "professional_accounting": 0.46285714285714286, "professional_law": 0.3127962085308057, "professional_medicine": 0.28191489361702127, "professional_psychology": 0.4224137931034483, "public_relations": 0.4540229885057471, "security_study": 0.37777777777777777, "sociology": 0.4557522123893805, "sports_science": 0.44242424242424244, "traditional_chinese_medicine": 0.31351351351351353, "virology": 0.4556213017751479, "world_history": 0.4782608695652174, "world_religions": 0.38125 } } }, "zbench": { "prompt_1": { "accuracy": 0.2727272727272727 }, "prompt_2": { "accuracy": 0.3939393939393939 }, "prompt_3": { "accuracy": 0.2727272727272727 }, "prompt_4": { "accuracy": 0.2727272727272727 }, "prompt_5": { "accuracy": 0.36363636363636365 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6113636363636363 }, "prompt_2": { "accuracy": 0.5772727272727273 }, "prompt_3": { "accuracy": 0.5681818181818182 }, "prompt_4": { "accuracy": 0.5568181818181818 }, "prompt_5": { "accuracy": 0.5681818181818182 } }, "ocnli": { "prompt_1": { "accuracy": 0.488135593220339 }, "prompt_2": { "accuracy": 0.46576271186440676 }, "prompt_3": { "accuracy": 0.49254237288135594 }, "prompt_4": { "accuracy": 0.47016949152542376 }, "prompt_5": { "accuracy": 0.471864406779661 } }, "c3": { "prompt_1": { "accuracy": 0.7554225878833208 }, "prompt_2": { "accuracy": 0.743455497382199 }, "prompt_3": { "accuracy": 0.7490650710545999 }, "prompt_4": { "accuracy": 0.7423335826477188 }, "prompt_5": { "accuracy": 0.74943904263276 } }, "dream": { "prompt_1": { "accuracy": 0.8554630083292504 }, "prompt_2": { "accuracy": 0.8637922586967173 }, "prompt_3": { "accuracy": 0.855952964233219 }, "prompt_4": { "accuracy": 0.8539931406173444 }, "prompt_5": { "accuracy": 0.8564429201371877 } }, "samsum": { "prompt_1": { "rouge1": 0.3166185955051697, "rouge2": 0.10917539998518976, "rougeL": 0.23837567739925294, "avg_rouge": 0.22138989096320413 }, "prompt_2": { "rouge1": 0.3461243017709684, "rouge2": 0.1196525210050174, "rougeL": 0.26254524986557515, "avg_rouge": 0.24277402421385363 }, "prompt_3": { "rouge1": 0.32016444453648835, "rouge2": 0.10572427291148377, "rougeL": 0.24004692556166563, "avg_rouge": 0.22197854766987923 }, "prompt_4": { "rouge1": 0.3245191995512825, "rouge2": 0.10936588200960218, "rougeL": 0.24459506910412812, "avg_rouge": 0.22616005022167093 }, "prompt_5": { "rouge1": 0.33512587005716804, "rouge2": 0.10875270341184964, "rougeL": 0.2504391336881377, "avg_rouge": 0.23143923571905178 } }, "dialogsum": { "prompt_1": { "rouge1": 0.2242608916043325, "rouge2": 0.05331708909380924, "rougeL": 0.16398737459191393, "avg_rouge": 0.14718845176335188 }, "prompt_2": { "rouge1": 0.2234401281407043, "rouge2": 0.05283218485911105, "rougeL": 0.16327769653604746, "avg_rouge": 0.1465166698452876 }, "prompt_3": { "rouge1": 0.22427132271501854, "rouge2": 0.052786421931518876, "rougeL": 0.16357874102702746, "avg_rouge": 0.14687882855785497 }, "prompt_4": { "rouge1": 0.22333528000915123, "rouge2": 0.05176884162561466, "rougeL": 0.16264925111044223, "avg_rouge": 0.14591779091506937 }, "prompt_5": { "rouge1": 0.22259900342738578, "rouge2": 0.052647787328002134, "rougeL": 0.1627164107373179, "avg_rouge": 0.14598773383090194 } }, "sst2": { "prompt_1": { "accuracy": 0.8864678899082569 }, "prompt_2": { "accuracy": 0.8910550458715596 }, "prompt_3": { "accuracy": 0.8830275229357798 }, "prompt_4": { "accuracy": 0.8830275229357798 }, "prompt_5": { "accuracy": 0.8589449541284404 } }, "cola": { "prompt_1": { "accuracy": 0.8092042186001918 }, "prompt_2": { "accuracy": 0.8024928092042186 }, "prompt_3": { "accuracy": 0.8082454458293384 }, "prompt_4": { "accuracy": 0.8130393096836049 }, "prompt_5": { "accuracy": 0.8130393096836049 } }, "qqp": { "prompt_1": { "accuracy": 0.626 }, "prompt_2": { "accuracy": 0.6235 }, "prompt_3": { "accuracy": 0.653 }, "prompt_4": { "accuracy": 0.612 }, "prompt_5": { "accuracy": 0.6185 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.6 } }, "qnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.6 } }, "mrpc": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.6 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "sailor_0_5b": { "model_size": "0.5B", "model_link": "https://huggingface.co/sail/Sailor-0.5B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.2542016806722689, "language_acc": { "Spanish": 0.253781512605042, "English": 0.23865546218487396, "Chinese": 0.2647058823529412, "Vietnamese": 0.2596638655462185 }, "consistency_score_2": 0.2838935574229691, "consistency_score_3": 0.0930672268907563, "consistency_score_4": 0.03277310924369748, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.3025210084033613, "Spanish,Chinese": 0.2915966386554622, "Spanish,Vietnamese": 0.2991596638655462, "English,Chinese": 0.2504201680672269, "English,Vietnamese": 0.2848739495798319, "Chinese,Vietnamese": 0.2747899159663866 }, "3_combine": { "Spanish,English,Chinese": 0.0865546218487395, "Spanish,English,Vietnamese": 0.11176470588235295, "Spanish,Chinese,Vietnamese": 0.09663865546218488, "English,Chinese,Vietnamese": 0.0773109243697479 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.03277310924369748 } }, "AC3_2": 0.2682284261462423, "AC3_3": 0.13625087054493334, "AC3_4": 0.05806070596054978 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.24761904761904763, "language_acc": { "Filipino": 0.28, "Vietnamese": 0.24, "Chinese": 0.26666666666666666, "Spanish": 0.21333333333333335, "Malay": 0.26, "Indonesian": 0.23333333333333334, "English": 0.24 }, "consistency_score_2": 0.2911111111111111, "consistency_score_3": 0.09828571428571428, "consistency_score_4": 0.03619047619047618, "consistency_score_5": 0.013333333333333332, "consistency_score_6": 0.00380952380952381, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.36, "Filipino,Chinese": 0.26666666666666666, "Filipino,Spanish": 0.3, "Filipino,Malay": 0.2866666666666667, "Filipino,Indonesian": 0.28, "Filipino,English": 0.2733333333333333, "Vietnamese,Chinese": 0.31333333333333335, "Vietnamese,Spanish": 0.24666666666666667, "Vietnamese,Malay": 0.32, "Vietnamese,Indonesian": 0.31333333333333335, "Vietnamese,English": 0.2866666666666667, "Chinese,Spanish": 0.3, "Chinese,Malay": 0.24666666666666667, "Chinese,Indonesian": 0.26, "Chinese,English": 0.31333333333333335, "Spanish,Malay": 0.30666666666666664, "Spanish,Indonesian": 0.2733333333333333, "Spanish,English": 0.2866666666666667, "Malay,Indonesian": 0.30666666666666664, "Malay,English": 0.31333333333333335, "Indonesian,English": 0.26 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.1, "Filipino,Vietnamese,Spanish": 0.08666666666666667, "Filipino,Vietnamese,Malay": 0.12666666666666668, "Filipino,Vietnamese,Indonesian": 0.12, "Filipino,Vietnamese,English": 0.09333333333333334, "Filipino,Chinese,Spanish": 0.08666666666666667, "Filipino,Chinese,Malay": 0.08666666666666667, "Filipino,Chinese,Indonesian": 0.06666666666666667, "Filipino,Chinese,English": 0.10666666666666667, "Filipino,Spanish,Malay": 0.09333333333333334, "Filipino,Spanish,Indonesian": 0.07333333333333333, "Filipino,Spanish,English": 0.08, "Filipino,Malay,Indonesian": 0.10666666666666667, "Filipino,Malay,English": 0.10666666666666667, "Filipino,Indonesian,English": 0.08666666666666667, "Vietnamese,Chinese,Spanish": 0.10666666666666667, "Vietnamese,Chinese,Malay": 0.08666666666666667, "Vietnamese,Chinese,Indonesian": 0.13333333333333333, "Vietnamese,Chinese,English": 0.13333333333333333, "Vietnamese,Spanish,Malay": 0.1, "Vietnamese,Spanish,Indonesian": 0.08, "Vietnamese,Spanish,English": 0.07333333333333333, "Vietnamese,Malay,Indonesian": 0.14, "Vietnamese,Malay,English": 0.1, "Vietnamese,Indonesian,English": 0.10666666666666667, "Chinese,Spanish,Malay": 0.09333333333333334, "Chinese,Spanish,Indonesian": 0.08, "Chinese,Spanish,English": 0.11333333333333333, "Chinese,Malay,Indonesian": 0.06, "Chinese,Malay,English": 0.12, "Chinese,Indonesian,English": 0.08, "Spanish,Malay,Indonesian": 0.1, "Spanish,Malay,English": 0.12666666666666668, "Spanish,Indonesian,English": 0.07333333333333333, "Malay,Indonesian,English": 0.11333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.03333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.03333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.04, "Filipino,Vietnamese,Chinese,English": 0.04, "Filipino,Vietnamese,Spanish,Malay": 0.02666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.02, "Filipino,Vietnamese,Spanish,English": 0.02, "Filipino,Vietnamese,Malay,Indonesian": 0.06, "Filipino,Vietnamese,Malay,English": 0.04, "Filipino,Vietnamese,Indonesian,English": 0.04, "Filipino,Chinese,Spanish,Malay": 0.02666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.013333333333333334, "Filipino,Chinese,Spanish,English": 0.03333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.02, "Filipino,Chinese,Malay,English": 0.04666666666666667, "Filipino,Chinese,Indonesian,English": 0.02, "Filipino,Spanish,Malay,Indonesian": 0.03333333333333333, "Filipino,Spanish,Malay,English": 0.02, "Filipino,Spanish,Indonesian,English": 0.013333333333333334, "Filipino,Malay,Indonesian,English": 0.05333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.04, "Vietnamese,Chinese,Spanish,Indonesian": 0.03333333333333333, "Vietnamese,Chinese,Spanish,English": 0.06, "Vietnamese,Chinese,Malay,Indonesian": 0.04, "Vietnamese,Chinese,Malay,English": 0.05333333333333334, "Vietnamese,Chinese,Indonesian,English": 0.06, "Vietnamese,Spanish,Malay,Indonesian": 0.04666666666666667, "Vietnamese,Spanish,Malay,English": 0.03333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.02, "Vietnamese,Malay,Indonesian,English": 0.04666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.02, "Chinese,Spanish,Malay,English": 0.06666666666666667, "Chinese,Spanish,Indonesian,English": 0.02666666666666667, "Chinese,Malay,Indonesian,English": 0.04, "Spanish,Malay,Indonesian,English": 0.04666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.02, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.02, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.013333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Malay,Indonesian,English": 0.02666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Chinese,Spanish,Malay,English": 0.013333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Chinese,Malay,Indonesian,English": 0.013333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.006666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.013333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.03333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.02, "Vietnamese,Chinese,Malay,Indonesian,English": 0.02666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.013333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.02 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.006666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.013333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 } }, "AC3_2": 0.2676095069234156, "AC3_3": 0.1407174323067073, "AC3_4": 0.06315116648461995, "AC3_5": 0.025304136243344523, "AC3_6": 0.007503607500623115, "AC3_7": 0.0 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.25081168831168826, "language_acc": { "Indonesian": 0.20454545454545456, "English": 0.2840909090909091, "Filipino": 0.2897727272727273, "Spanish": 0.26704545454545453, "Chinese": 0.23295454545454544, "Malay": 0.2159090909090909, "Vietnamese": 0.26136363636363635 }, "consistency_score_2": 0.28977272727272724, "consistency_score_3": 0.09399350649350649, "consistency_score_4": 0.032629870129870134, "consistency_score_5": 0.011634199134199132, "consistency_score_6": 0.004058441558441559, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.25, "Indonesian,Filipino": 0.26704545454545453, "Indonesian,Spanish": 0.3125, "Indonesian,Chinese": 0.22727272727272727, "Indonesian,Malay": 0.2556818181818182, "Indonesian,Vietnamese": 0.36363636363636365, "English,Filipino": 0.32386363636363635, "English,Spanish": 0.29545454545454547, "English,Chinese": 0.23863636363636365, "English,Malay": 0.3125, "English,Vietnamese": 0.32386363636363635, "Filipino,Spanish": 0.2784090909090909, "Filipino,Chinese": 0.2897727272727273, "Filipino,Malay": 0.2784090909090909, "Filipino,Vietnamese": 0.2897727272727273, "Spanish,Chinese": 0.26136363636363635, "Spanish,Malay": 0.2897727272727273, "Spanish,Vietnamese": 0.3465909090909091, "Chinese,Malay": 0.30113636363636365, "Chinese,Vietnamese": 0.26704545454545453, "Malay,Vietnamese": 0.3125 }, "3_combine": { "Indonesian,English,Filipino": 0.08522727272727272, "Indonesian,English,Spanish": 0.0625, "Indonesian,English,Chinese": 0.045454545454545456, "Indonesian,English,Malay": 0.09090909090909091, "Indonesian,English,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish": 0.07954545454545454, "Indonesian,Filipino,Chinese": 0.07386363636363637, "Indonesian,Filipino,Malay": 0.07954545454545454, "Indonesian,Filipino,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Chinese": 0.09090909090909091, "Indonesian,Spanish,Malay": 0.10227272727272728, "Indonesian,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,Chinese,Malay": 0.045454545454545456, "Indonesian,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish": 0.10795454545454546, "English,Filipino,Chinese": 0.09659090909090909, "English,Filipino,Malay": 0.11363636363636363, "English,Filipino,Vietnamese": 0.125, "English,Spanish,Chinese": 0.07954545454545454, "English,Spanish,Malay": 0.09659090909090909, "English,Spanish,Vietnamese": 0.10227272727272728, "English,Chinese,Malay": 0.09659090909090909, "English,Chinese,Vietnamese": 0.09659090909090909, "English,Malay,Vietnamese": 0.11931818181818182, "Filipino,Spanish,Chinese": 0.07954545454545454, "Filipino,Spanish,Malay": 0.07386363636363637, "Filipino,Spanish,Vietnamese": 0.09090909090909091, "Filipino,Chinese,Malay": 0.10227272727272728, "Filipino,Chinese,Vietnamese": 0.10227272727272728, "Filipino,Malay,Vietnamese": 0.07954545454545454, "Spanish,Chinese,Malay": 0.09090909090909091, "Spanish,Chinese,Vietnamese": 0.09659090909090909, "Spanish,Malay,Vietnamese": 0.125, "Chinese,Malay,Vietnamese": 0.11363636363636363 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.028409090909090908, "Indonesian,English,Filipino,Chinese": 0.022727272727272728, "Indonesian,English,Filipino,Malay": 0.03409090909090909, "Indonesian,English,Filipino,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Chinese": 0.022727272727272728, "Indonesian,English,Spanish,Malay": 0.028409090909090908, "Indonesian,English,Spanish,Vietnamese": 0.03409090909090909, "Indonesian,English,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay": 0.022727272727272728, "Indonesian,Filipino,Spanish,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Chinese,Malay": 0.028409090909090908, "Indonesian,Filipino,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.017045454545454544, "Indonesian,Spanish,Chinese,Malay": 0.03409090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese": 0.03977272727272727, "English,Filipino,Spanish,Malay": 0.03977272727272727, "English,Filipino,Spanish,Vietnamese": 0.03977272727272727, "English,Filipino,Chinese,Malay": 0.03977272727272727, "English,Filipino,Chinese,Vietnamese": 0.05113636363636364, "English,Filipino,Malay,Vietnamese": 0.03977272727272727, "English,Spanish,Chinese,Malay": 0.028409090909090908, "English,Spanish,Chinese,Vietnamese": 0.03409090909090909, "English,Spanish,Malay,Vietnamese": 0.03409090909090909, "English,Chinese,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese,Malay": 0.022727272727272728, "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, "Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Spanish,Malay,Vietnamese": 0.011363636363636364, "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, "AC3_2": 0.26888820633846605, "AC3_3": 0.1367419656217994, "AC3_4": 0.05774702099984339, "AC3_5": 0.022236912563491164, "AC3_6": 0.007987633382588557, "AC3_7": 0.0 } }, "sg_eval": { "prompt_1": { "accuracy": 0.17475728155339806 } }, "cn_eval": { "prompt_1": { "accuracy": 0.24761904761904763 } }, "us_eval": { "prompt_1": { "accuracy": 0.2803738317757009 } }, "ph_eval": { "prompt_1": { "accuracy": 0.29, "category_acc": { "brand": 0.2, "demographics": 0.4, "biology": 0.2, "history": 0.5333333333333333, "literature": 0.2, "politics": 0.5, "culture": 0.2, "film": 0.2, "law": 0.2, "geography": 0.2 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.11531135447491982 } }, "indommlu": { "prompt_1": { "accuracy": 0.23452833967554576, "category_acc": { "History": 0.19076305220883535, "Geography": 0.21836734693877552, "Lampungic": 0.2585034013605442, "Social science": 0.25375626043405675, "Balinese": 0.27176220806794055, "Makassarese": 0.2903225806451613, "Banjarese": 0.25, "Chemistry": 0.2029197080291971, "Biology": 0.20946745562130178, "Science": 0.24561403508771928, "Christian religion": 0.23880597014925373, "Art": 0.2529118136439268, "Islam religion": 0.23897581792318634, "Hindu religion": 0.30666666666666664, "Madurese": 0.2576271186440678, "Sport": 0.17567567567567569, "Indonesian language": 0.23225404732254049, "Physics": 0.2222222222222222, "Minangkabau culture": 0.2964824120603015, "Dayak language": 0.22018348623853212, "Sociology": 0.20161290322580644, "Economy": 0.18442622950819673, "Sundanese": 0.2549697493517718, "Javanese": 0.25, "Civic education": 0.23032904148783978 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.12903915185649562 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.12569739315639059 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.039003259240151175 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.13061525238014365 } }, "mmlu": { "prompt_1": { "accuracy": 0.2660443407234539 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.24883804075795496, "category_acc": { "high_school_european_history": 0.2621951219512195, "business_ethics": 0.29292929292929293, "clinical_knowledge": 0.21212121212121213, "medical_genetics": 0.23232323232323232, "high_school_us_history": 0.28078817733990147, "high_school_physics": 0.24666666666666667, "high_school_world_history": 0.2457627118644068, "virology": 0.24848484848484848, "high_school_microeconomics": 0.25316455696202533, "econometrics": 0.20353982300884957, "college_computer_science": 0.29292929292929293, "high_school_biology": 0.2297734627831715, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.2846975088967972, "philosophy": 0.25483870967741934, "professional_medicine": 0.22509225092250923, "nutrition": 0.25245901639344265, "global_facts": 0.2727272727272727, "machine_learning": 0.23423423423423423, "security_studies": 0.21721311475409835, "public_relations": 0.22018348623853212, "professional_psychology": 0.27495908346972175, "prehistory": 0.2260061919504644, "anatomy": 0.22388059701492538, "human_sexuality": 0.26153846153846155, "college_medicine": 0.22674418604651161, "high_school_government_and_politics": 0.2916666666666667, "college_chemistry": 0.26262626262626265, "logical_fallacies": 0.24074074074074073, "high_school_geography": 0.23857868020304568, "elementary_mathematics": 0.21220159151193635, "human_aging": 0.2747747747747748, "college_mathematics": 0.20202020202020202, "high_school_psychology": 0.23897058823529413, "formal_logic": 0.264, "high_school_statistics": 0.32558139534883723, "international_law": 0.26666666666666666, "high_school_mathematics": 0.24907063197026022, "high_school_computer_science": 0.30303030303030304, "conceptual_physics": 0.24358974358974358, "miscellaneous": 0.22506393861892582, "high_school_chemistry": 0.2524752475247525, "marketing": 0.33476394849785407, "professional_law": 0.23222439660795824, "management": 0.2647058823529412, "college_physics": 0.16831683168316833, "jurisprudence": 0.205607476635514, "world_religions": 0.3235294117647059, "sociology": 0.24, "us_foreign_policy": 0.3333333333333333, "high_school_macroeconomics": 0.2390745501285347, "computer_security": 0.2222222222222222, "moral_scenarios": 0.2483221476510067, "moral_disputes": 0.23478260869565218, "electrical_engineering": 0.2708333333333333, "astronomy": 0.33112582781456956, "college_biology": 0.2867132867132867 } } }, "c_eval": { "prompt_1": { "accuracy": 0.23402674591381872 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.24221668742216687, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.25, "computer_architecture": 0.2692307692307692, "college_programming": 0.21428571428571427, "college_physics": 0.08333333333333333, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.16666666666666666, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.125, "high_school_chemistry": 0.25, "high_school_biology": 0.375, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.23076923076923078, "middle_school_physics": 0.25, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.3333333333333333, "business_administration": 0.21052631578947367, "marxism": 0.25, "mao_zedong_thought": 0.2413793103448276, "education_science": 0.17647058823529413, "teacher_qualification": 0.2857142857142857, "high_school_politics": 0.3333333333333333, "high_school_geography": 0.375, "middle_school_politics": 0.4230769230769231, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.10714285714285714, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.18518518518518517, "law": 0.2413793103448276, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.3157894736842105, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.25, "high_school_history": 0.2, "middle_school_history": 0.3333333333333333, "civil_servant": 0.23076923076923078, "sports_science": 0.25, "plant_protection": 0.25925925925925924, "basic_medicine": 0.041666666666666664, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.21568627450980393, "accountant": 0.2962962962962963, "fire_engineer": 0.1388888888888889, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.18518518518518517, "physician": 0.12962962962962962 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.27598566308243727 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2515973061647384, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.22972972972972974, "ancient_chinese": 0.25, "arts": 0.30625, "astronomy": 0.24242424242424243, "business_ethics": 0.28708133971291866, "chinese_civil_service_exam": 0.2875, "chinese_driving_rule": 0.2748091603053435, "chinese_food_culture": 0.18382352941176472, "chinese_foreign_policy": 0.27102803738317754, "chinese_history": 0.2260061919504644, "chinese_literature": 0.22058823529411764, "chinese_teacher_qualification": 0.24022346368715083, "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.18867924528301888, "college_education": 0.308411214953271, "college_engineering_hydrology": 0.2358490566037736, "college_law": 0.26851851851851855, "college_mathematics": 0.13333333333333333, "college_medical_statistics": 0.22641509433962265, "college_medicine": 0.26373626373626374, "computer_science": 0.20588235294117646, "computer_security": 0.23976608187134502, "conceptual_physics": 0.25170068027210885, "construction_project_management": 0.2733812949640288, "economics": 0.2389937106918239, "education": 0.2331288343558282, "electrical_engineering": 0.2441860465116279, "elementary_chinese": 0.24206349206349206, "elementary_commonsense": 0.2828282828282828, "elementary_information_and_technology": 0.25630252100840334, "elementary_mathematics": 0.28695652173913044, "ethnology": 0.2518518518518518, "food_science": 0.23776223776223776, "genetics": 0.26136363636363635, "global_facts": 0.19463087248322147, "high_school_biology": 0.21301775147928995, "high_school_chemistry": 0.22727272727272727, "high_school_geography": 0.2627118644067797, "high_school_mathematics": 0.2073170731707317, "high_school_physics": 0.3, "high_school_politics": 0.23776223776223776, "human_sexuality": 0.21428571428571427, "international_law": 0.22702702702702704, "journalism": 0.27325581395348836, "jurisprudence": 0.22871046228710462, "legal_and_moral_basis": 0.2757009345794392, "logical": 0.21951219512195122, "machine_learning": 0.22131147540983606, "management": 0.2761904761904762, "marketing": 0.24444444444444444, "marxist_theory": 0.291005291005291, "modern_chinese": 0.2413793103448276, "nutrition": 0.2, "philosophy": 0.18095238095238095, "professional_accounting": 0.3028571428571429, "professional_law": 0.23222748815165878, "professional_medicine": 0.23138297872340424, "professional_psychology": 0.31896551724137934, "public_relations": 0.28160919540229884, "security_study": 0.2740740740740741, "sociology": 0.3008849557522124, "sports_science": 0.32727272727272727, "traditional_chinese_medicine": 0.20540540540540542, "virology": 0.30177514792899407, "world_history": 0.2857142857142857, "world_religions": 0.3125 } } }, "zbench": { "prompt_1": { "accuracy": 0.18181818181818182 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.16590909090909092 } }, "ocnli": { "prompt_1": { "accuracy": 0.3311864406779661 } }, "c3": { "prompt_1": { "accuracy": 0.2629020194465221 } }, "dream": { "prompt_1": { "accuracy": 0.329250367466928 } }, "samsum": { "prompt_1": { "rouge1": 0.12898713332403366, "rouge2": 0.04349609409834999, "rougeL": 0.10177675146188521, "avg_rouge": 0.09141999296142295 } }, "dialogsum": { "prompt_1": { "rouge1": 0.13775292661101599, "rouge2": 0.04111655319766571, "rougeL": 0.10976116581390764, "avg_rouge": 0.09621021520752977 } }, "sst2": { "prompt_1": { "accuracy": 0.5080275229357798 } }, "cola": { "prompt_1": { "accuracy": 0.5148609779482263 } }, "qqp": { "prompt_1": { "accuracy": 0.498 } }, "mnli": { "prompt_1": { "accuracy": 0.2 } }, "qnli": { "prompt_1": { "accuracy": 0.6 } }, "wnli": { "prompt_1": { "accuracy": 0.3 } }, "rte": { "prompt_1": { "accuracy": 0.1 } }, "mrpc": { "prompt_1": { "accuracy": 0.7 } } } }, "sailor_1_8b": { "model_size": "1.8B", "model_link": "https://huggingface.co/sail/Sailor-1.8B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.26, "language_acc": { "Filipino": 0.20666666666666667, "Vietnamese": 0.26666666666666666, "Chinese": 0.30666666666666664, "Spanish": 0.25333333333333335, "Malay": 0.26, "Indonesian": 0.28, "English": 0.24666666666666667 }, "consistency_score_2": 0.3946031746031746, "consistency_score_3": 0.18895238095238098, "consistency_score_4": 0.10114285714285713, "consistency_score_5": 0.056507936507936514, "consistency_score_6": 0.030476190476190473, "consistency_score_7": 0.013333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.36, "Filipino,Chinese": 0.31333333333333335, "Filipino,Spanish": 0.3333333333333333, "Filipino,Malay": 0.4066666666666667, "Filipino,Indonesian": 0.41333333333333333, "Filipino,English": 0.34, "Vietnamese,Chinese": 0.5, "Vietnamese,Spanish": 0.37333333333333335, "Vietnamese,Malay": 0.36666666666666664, "Vietnamese,Indonesian": 0.42, "Vietnamese,English": 0.44, "Chinese,Spanish": 0.32666666666666666, "Chinese,Malay": 0.3933333333333333, "Chinese,Indonesian": 0.44666666666666666, "Chinese,English": 0.44666666666666666, "Spanish,Malay": 0.34, "Spanish,Indonesian": 0.35333333333333333, "Spanish,English": 0.3466666666666667, "Malay,Indonesian": 0.4666666666666667, "Malay,English": 0.4666666666666667, "Indonesian,English": 0.43333333333333335 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.18666666666666668, "Filipino,Vietnamese,Spanish": 0.13333333333333333, "Filipino,Vietnamese,Malay": 0.19333333333333333, "Filipino,Vietnamese,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,English": 0.16666666666666666, "Filipino,Chinese,Spanish": 0.09333333333333334, "Filipino,Chinese,Malay": 0.16666666666666666, "Filipino,Chinese,Indonesian": 0.18, "Filipino,Chinese,English": 0.13333333333333333, "Filipino,Spanish,Malay": 0.15333333333333332, "Filipino,Spanish,Indonesian": 0.15333333333333332, "Filipino,Spanish,English": 0.12, "Filipino,Malay,Indonesian": 0.22, "Filipino,Malay,English": 0.18666666666666668, "Filipino,Indonesian,English": 0.18, "Vietnamese,Chinese,Spanish": 0.2, "Vietnamese,Chinese,Malay": 0.24, "Vietnamese,Chinese,Indonesian": 0.26, "Vietnamese,Chinese,English": 0.2733333333333333, "Vietnamese,Spanish,Malay": 0.16666666666666666, "Vietnamese,Spanish,Indonesian": 0.18, "Vietnamese,Spanish,English": 0.16, "Vietnamese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Malay,English": 0.22, "Vietnamese,Indonesian,English": 0.24, "Chinese,Spanish,Malay": 0.16, "Chinese,Spanish,Indonesian": 0.16, "Chinese,Spanish,English": 0.16, "Chinese,Malay,Indonesian": 0.24666666666666667, "Chinese,Malay,English": 0.23333333333333334, "Chinese,Indonesian,English": 0.22666666666666666, "Spanish,Malay,Indonesian": 0.18666666666666668, "Spanish,Malay,English": 0.18, "Spanish,Indonesian,English": 0.16666666666666666, "Malay,Indonesian,English": 0.26666666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.06, "Filipino,Vietnamese,Chinese,Malay": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,English": 0.1, "Filipino,Vietnamese,Spanish,Malay": 0.08666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.08, "Filipino,Vietnamese,Spanish,English": 0.06, "Filipino,Vietnamese,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay": 0.06666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.05333333333333334, "Filipino,Chinese,Spanish,English": 0.04, "Filipino,Chinese,Malay,Indonesian": 0.11333333333333333, "Filipino,Chinese,Malay,English": 0.08666666666666667, "Filipino,Chinese,Indonesian,English": 0.07333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.08, "Filipino,Spanish,Malay,English": 0.08, "Filipino,Spanish,Indonesian,English": 0.07333333333333333, "Filipino,Malay,Indonesian,English": 0.12, "Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Vietnamese,Chinese,Spanish,English": 0.11333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.10666666666666667, "Vietnamese,Spanish,Malay,English": 0.1, "Vietnamese,Spanish,Indonesian,English": 0.09333333333333334, "Vietnamese,Malay,Indonesian,English": 0.14666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, "Chinese,Spanish,Malay,English": 0.10666666666666667, "Chinese,Spanish,Indonesian,English": 0.08666666666666667, "Chinese,Malay,Indonesian,English": 0.14, "Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.03333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.03333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.06, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.04666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.05333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.04, "Filipino,Vietnamese,Malay,Indonesian,English": 0.07333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.04, "Filipino,Chinese,Spanish,Malay,English": 0.04, "Filipino,Chinese,Spanish,Indonesian,English": 0.02, "Filipino,Chinese,Malay,Indonesian,English": 0.05333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.05333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.08, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, "Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.06666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.02666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.03333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.013333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.04, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.03333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.02, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.013333333333333334 } }, "AC3_2": 0.31346265756608116, "AC3_3": 0.21885447598010244, "AC3_4": 0.14563291135207948, "AC3_5": 0.0928385155173078, "AC3_6": 0.05455737703039828, "AC3_7": 0.0253658536492564 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.25487012987012986, "language_acc": { "Indonesian": 0.2556818181818182, "English": 0.24431818181818182, "Filipino": 0.23863636363636365, "Spanish": 0.26704545454545453, "Chinese": 0.26704545454545453, "Malay": 0.2556818181818182, "Vietnamese": 0.2556818181818182 }, "consistency_score_2": 0.5722402597402597, "consistency_score_3": 0.39074675324675323, "consistency_score_4": 0.27516233766233766, "consistency_score_5": 0.19318181818181818, "consistency_score_6": 0.13392857142857145, "consistency_score_7": 0.09090909090909091, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7045454545454546, "Indonesian,Filipino": 0.4034090909090909, "Indonesian,Spanish": 0.5454545454545454, "Indonesian,Chinese": 0.625, "Indonesian,Malay": 0.6534090909090909, "Indonesian,Vietnamese": 0.6704545454545454, "English,Filipino": 0.4602272727272727, "English,Spanish": 0.5795454545454546, "English,Chinese": 0.6534090909090909, "English,Malay": 0.7329545454545454, "English,Vietnamese": 0.75, "Filipino,Spanish": 0.4318181818181818, "Filipino,Chinese": 0.4090909090909091, "Filipino,Malay": 0.39204545454545453, "Filipino,Vietnamese": 0.4090909090909091, "Spanish,Chinese": 0.5056818181818182, "Spanish,Malay": 0.5113636363636364, "Spanish,Vietnamese": 0.5568181818181818, "Chinese,Malay": 0.625, "Chinese,Vietnamese": 0.6704545454545454, "Malay,Vietnamese": 0.7272727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.3181818181818182, "Indonesian,English,Spanish": 0.4318181818181818, "Indonesian,English,Chinese": 0.5170454545454546, "Indonesian,English,Malay": 0.5568181818181818, "Indonesian,English,Vietnamese": 0.5681818181818182, "Indonesian,Filipino,Spanish": 0.24431818181818182, "Indonesian,Filipino,Chinese": 0.2784090909090909, "Indonesian,Filipino,Malay": 0.2727272727272727, "Indonesian,Filipino,Vietnamese": 0.2897727272727273, "Indonesian,Spanish,Chinese": 0.3693181818181818, "Indonesian,Spanish,Malay": 0.3806818181818182, "Indonesian,Spanish,Vietnamese": 0.4147727272727273, "Indonesian,Chinese,Malay": 0.4772727272727273, "Indonesian,Chinese,Vietnamese": 0.5, "Indonesian,Malay,Vietnamese": 0.5340909090909091, "English,Filipino,Spanish": 0.2840909090909091, "English,Filipino,Chinese": 0.30113636363636365, "English,Filipino,Malay": 0.3352272727272727, "English,Filipino,Vietnamese": 0.3409090909090909, "English,Spanish,Chinese": 0.39204545454545453, "English,Spanish,Malay": 0.42613636363636365, "English,Spanish,Vietnamese": 0.4602272727272727, "English,Chinese,Malay": 0.5284090909090909, "English,Chinese,Vietnamese": 0.5511363636363636, "English,Malay,Vietnamese": 0.6136363636363636, "Filipino,Spanish,Chinese": 0.23863636363636365, "Filipino,Spanish,Malay": 0.22727272727272727, "Filipino,Spanish,Vietnamese": 0.2556818181818182, "Filipino,Chinese,Malay": 0.26704545454545453, "Filipino,Chinese,Vietnamese": 0.30113636363636365, "Filipino,Malay,Vietnamese": 0.30113636363636365, "Spanish,Chinese,Malay": 0.35795454545454547, "Spanish,Chinese,Vietnamese": 0.4034090909090909, "Spanish,Malay,Vietnamese": 0.4090909090909091, "Chinese,Malay,Vietnamese": 0.5284090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.21022727272727273, "Indonesian,English,Filipino,Chinese": 0.23295454545454544, "Indonesian,English,Filipino,Malay": 0.23863636363636365, "Indonesian,English,Filipino,Vietnamese": 0.24431818181818182, "Indonesian,English,Spanish,Chinese": 0.3125, "Indonesian,English,Spanish,Malay": 0.32954545454545453, "Indonesian,English,Spanish,Vietnamese": 0.36363636363636365, "Indonesian,English,Chinese,Malay": 0.4147727272727273, "Indonesian,English,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,English,Malay,Vietnamese": 0.4659090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.17613636363636365, "Indonesian,Filipino,Spanish,Malay": 0.16477272727272727, "Indonesian,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Chinese,Malay": 0.19886363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Chinese,Malay": 0.2727272727272727, "Indonesian,Spanish,Chinese,Vietnamese": 0.3125, "Indonesian,Spanish,Malay,Vietnamese": 0.3125, "Indonesian,Chinese,Malay,Vietnamese": 0.4090909090909091, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.20454545454545456, "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, "English,Filipino,Chinese,Malay": 0.2215909090909091, "English,Filipino,Chinese,Vietnamese": 0.2556818181818182, "English,Filipino,Malay,Vietnamese": 0.26704545454545453, "English,Spanish,Chinese,Malay": 0.3068181818181818, "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, "English,Spanish,Malay,Vietnamese": 0.36363636363636365, "English,Chinese,Malay,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.23863636363636365, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.35795454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "English,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 } }, "AC3_2": 0.35266622480219295, "AC3_3": 0.30851013452470777, "AC3_4": 0.264627791760101, "AC3_5": 0.21977931483895832, "AC3_6": 0.1755890247087898, "AC3_7": 0.13401621848450304 } }, "sg_eval": { "prompt_1": { "accuracy": 0.23300970873786409 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2857142857142857 } }, "us_eval": { "prompt_1": { "accuracy": 0.32710280373831774 } }, "ph_eval": { "prompt_1": { "accuracy": 0.22, "category_acc": { "brand": 0.0, "demographics": 0.2, "biology": 0.4, "history": 0.06666666666666667, "literature": 0.2, "politics": 0.3, "culture": 0.1, "film": 0.5, "law": 0.3, "geography": 0.2 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.24634338960756805 } }, "indommlu": { "prompt_1": { "accuracy": 0.2410708324988317, "category_acc": { "History": 0.20883534136546184, "Geography": 0.23877551020408164, "Lampungic": 0.2653061224489796, "Social science": 0.25542570951585974, "Balinese": 0.25690021231422505, "Makassarese": 0.26881720430107525, "Banjarese": 0.2847222222222222, "Chemistry": 0.21897810218978103, "Biology": 0.20946745562130178, "Science": 0.26109391124871, "Christian religion": 0.24875621890547264, "Art": 0.23627287853577372, "Islam religion": 0.24324324324324326, "Hindu religion": 0.18666666666666668, "Madurese": 0.2542372881355932, "Sport": 0.22972972972972974, "Indonesian language": 0.2636986301369863, "Physics": 0.18585858585858586, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.23853211009174313, "Sociology": 0.18346774193548387, "Economy": 0.2069672131147541, "Sundanese": 0.24719101123595505, "Javanese": 0.2439516129032258, "Civic education": 0.24892703862660945 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.3287443508742123 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.2659669890064635 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.17170080951799402 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.32481248192705475 } }, "mmlu": { "prompt_1": { "accuracy": 0.2718786464410735 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.2702895959957097, "category_acc": { "high_school_european_history": 0.2621951219512195, "business_ethics": 0.29292929292929293, "clinical_knowledge": 0.3522727272727273, "medical_genetics": 0.35353535353535354, "high_school_us_history": 0.270935960591133, "high_school_physics": 0.34, "high_school_world_history": 0.2033898305084746, "virology": 0.2545454545454545, "high_school_microeconomics": 0.29957805907172996, "econometrics": 0.22123893805309736, "college_computer_science": 0.25252525252525254, "high_school_biology": 0.35275080906148865, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.2313167259786477, "philosophy": 0.2709677419354839, "professional_medicine": 0.3874538745387454, "nutrition": 0.26885245901639343, "global_facts": 0.20202020202020202, "machine_learning": 0.18018018018018017, "security_studies": 0.3442622950819672, "public_relations": 0.23853211009174313, "professional_psychology": 0.25859247135842883, "prehistory": 0.25696594427244585, "anatomy": 0.23134328358208955, "human_sexuality": 0.26153846153846155, "college_medicine": 0.313953488372093, "high_school_government_and_politics": 0.3333333333333333, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.2037037037037037, "high_school_geography": 0.3553299492385787, "elementary_mathematics": 0.246684350132626, "human_aging": 0.17567567567567569, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.31985294117647056, "formal_logic": 0.28, "high_school_statistics": 0.3674418604651163, "international_law": 0.16666666666666666, "high_school_mathematics": 0.2788104089219331, "high_school_computer_science": 0.2222222222222222, "conceptual_physics": 0.2606837606837607, "miscellaneous": 0.21994884910485935, "high_school_chemistry": 0.23267326732673269, "marketing": 0.22317596566523606, "professional_law": 0.2648401826484018, "management": 0.29411764705882354, "college_physics": 0.31683168316831684, "jurisprudence": 0.29906542056074764, "world_religions": 0.20588235294117646, "sociology": 0.245, "us_foreign_policy": 0.32323232323232326, "high_school_macroeconomics": 0.3161953727506427, "computer_security": 0.25252525252525254, "moral_scenarios": 0.2595078299776286, "moral_disputes": 0.2318840579710145, "electrical_engineering": 0.2569444444444444, "astronomy": 0.2847682119205298, "college_biology": 0.21678321678321677 } } }, "c_eval": { "prompt_1": { "accuracy": 0.2563150074294205 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.273972602739726, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4166666666666667, "computer_architecture": 0.3076923076923077, "college_programming": 0.19047619047619047, "college_physics": 0.20833333333333334, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.4166666666666667, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.25, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.2833333333333333, "business_administration": 0.23684210526315788, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.2413793103448276, "education_science": 0.4117647058823529, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.19230769230769232, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.25, "logic": 0.37037037037037035, "law": 0.1724137931034483, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.15789473684210525, "professional_tour_guide": 0.23529411764705882, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.25, "high_school_history": 0.28, "middle_school_history": 0.2962962962962963, "civil_servant": 0.19230769230769232, "sports_science": 0.25, "plant_protection": 0.2962962962962963, "basic_medicine": 0.4583333333333333, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.25925925925925924, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.25925925925925924, "physician": 0.35185185185185186 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.25448028673835127 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.26144016577447765, "category_acc": { "agronomy": 0.2781065088757396, "anatomy": 0.30405405405405406, "ancient_chinese": 0.2621951219512195, "arts": 0.26875, "astronomy": 0.24848484848484848, "business_ethics": 0.27751196172248804, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.29770992366412213, "chinese_food_culture": 0.22794117647058823, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.21981424148606812, "chinese_literature": 0.2696078431372549, "chinese_teacher_qualification": 0.2737430167597765, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.3584905660377358, "college_education": 0.17757009345794392, "college_engineering_hydrology": 0.3113207547169811, "college_law": 0.3055555555555556, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2454212454212454, "computer_science": 0.2647058823529412, "computer_security": 0.2573099415204678, "conceptual_physics": 0.21768707482993196, "construction_project_management": 0.302158273381295, "economics": 0.24528301886792453, "education": 0.25766871165644173, "electrical_engineering": 0.22674418604651161, "elementary_chinese": 0.26587301587301587, "elementary_commonsense": 0.2878787878787879, "elementary_information_and_technology": 0.2647058823529412, "elementary_mathematics": 0.21304347826086956, "ethnology": 0.2740740740740741, "food_science": 0.2097902097902098, "genetics": 0.26136363636363635, "global_facts": 0.2214765100671141, "high_school_biology": 0.2958579881656805, "high_school_chemistry": 0.24242424242424243, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.2, "high_school_politics": 0.3006993006993007, "human_sexuality": 0.2857142857142857, "international_law": 0.25405405405405407, "journalism": 0.2616279069767442, "jurisprudence": 0.25790754257907544, "legal_and_moral_basis": 0.29906542056074764, "logical": 0.24390243902439024, "machine_learning": 0.2459016393442623, "management": 0.2571428571428571, "marketing": 0.2833333333333333, "marxist_theory": 0.2857142857142857, "modern_chinese": 0.23275862068965517, "nutrition": 0.25517241379310346, "philosophy": 0.29523809523809524, "professional_accounting": 0.28, "professional_law": 0.26066350710900477, "professional_medicine": 0.2526595744680851, "professional_psychology": 0.27155172413793105, "public_relations": 0.27011494252873564, "security_study": 0.2740740740740741, "sociology": 0.3141592920353982, "sports_science": 0.23030303030303031, "traditional_chinese_medicine": 0.2918918918918919, "virology": 0.26627218934911245, "world_history": 0.2236024844720497, "world_religions": 0.2625 } } }, "zbench": { "prompt_1": { "accuracy": 0.2727272727272727 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.18409090909090908 } }, "ocnli": { "prompt_1": { "accuracy": 0.3403389830508475 } }, "c3": { "prompt_1": { "accuracy": 0.28085265519820496 } }, "dream": { "prompt_1": { "accuracy": 0.3375796178343949 } }, "samsum": { "prompt_1": { "rouge1": 0.3764256748130376, "rouge2": 0.14284717779230788, "rougeL": 0.30317993386561815, "avg_rouge": 0.27415092882365455 } }, "dialogsum": { "prompt_1": { "rouge1": 0.37555740903295204, "rouge2": 0.12309897604398988, "rougeL": 0.3049472937088673, "avg_rouge": 0.2678678929286031 } }, "sst2": { "prompt_1": { "accuracy": 0.5424311926605505 } }, "cola": { "prompt_1": { "accuracy": 0.49185043144774687 } }, "qqp": { "prompt_1": { "accuracy": 0.4985 } }, "mnli": { "prompt_1": { "accuracy": 0.3 } }, "qnli": { "prompt_1": { "accuracy": 0.7 } }, "wnli": { "prompt_1": { "accuracy": 0.2 } }, "rte": { "prompt_1": { "accuracy": 0.3 } }, "mrpc": { "prompt_1": { "accuracy": 0.5 } } } }, "sailor_4b": { "model_size": "4B", "model_link": "https://huggingface.co/sail/Sailor-4B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.4957983193277311, "language_acc": { "Spanish": 0.46554621848739497, "English": 0.5092436974789916, "Chinese": 0.5016806722689076, "Vietnamese": 0.5067226890756302 }, "consistency_score_2": 0.6123249299719888, "consistency_score_3": 0.4619747899159664, "consistency_score_4": 0.37815126050420167, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.607563025210084, "Spanish,Chinese": 0.5873949579831933, "Spanish,Vietnamese": 0.6310924369747899, "English,Chinese": 0.6327731092436975, "English,Vietnamese": 0.6235294117647059, "Chinese,Vietnamese": 0.5915966386554622 }, "3_combine": { "Spanish,English,Chinese": 0.45966386554621846, "Spanish,English,Vietnamese": 0.4722689075630252, "Spanish,Chinese,Vietnamese": 0.44789915966386556, "English,Chinese,Vietnamese": 0.4680672268907563 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.37815126050420167 } }, "AC3_2": 0.5479348462854469, "AC3_3": 0.47828931962616705, "AC3_4": 0.42905623783067337 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.37523809523809526, "language_acc": { "Filipino": 0.3333333333333333, "Vietnamese": 0.36666666666666664, "Chinese": 0.3933333333333333, "Spanish": 0.36666666666666664, "Malay": 0.38666666666666666, "Indonesian": 0.35333333333333333, "English": 0.4266666666666667 }, "consistency_score_2": 0.5292063492063492, "consistency_score_3": 0.34228571428571425, "consistency_score_4": 0.24476190476190482, "consistency_score_5": 0.18571428571428572, "consistency_score_6": 0.14666666666666667, "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.58, "Filipino,Chinese": 0.5133333333333333, "Filipino,Spanish": 0.5266666666666666, "Filipino,Malay": 0.4866666666666667, "Filipino,Indonesian": 0.5066666666666667, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.6, "Vietnamese,Spanish": 0.5466666666666666, "Vietnamese,Malay": 0.5333333333333333, "Vietnamese,Indonesian": 0.5466666666666666, "Vietnamese,English": 0.56, "Chinese,Spanish": 0.58, "Chinese,Malay": 0.52, "Chinese,Indonesian": 0.54, "Chinese,English": 0.6066666666666667, "Spanish,Malay": 0.47333333333333333, "Spanish,Indonesian": 0.5066666666666667, "Spanish,English": 0.49333333333333335, "Malay,Indonesian": 0.56, "Malay,English": 0.47333333333333333, "Indonesian,English": 0.49333333333333335 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.37333333333333335, "Filipino,Vietnamese,Spanish": 0.36, "Filipino,Vietnamese,Malay": 0.37333333333333335, "Filipino,Vietnamese,Indonesian": 0.36, "Filipino,Vietnamese,English": 0.34, "Filipino,Chinese,Spanish": 0.36, "Filipino,Chinese,Malay": 0.30666666666666664, "Filipino,Chinese,Indonesian": 0.32, "Filipino,Chinese,English": 0.34, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.31333333333333335, "Filipino,Malay,Indonesian": 0.32666666666666666, "Filipino,Malay,English": 0.28, "Filipino,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish": 0.3933333333333333, "Vietnamese,Chinese,Malay": 0.37333333333333335, "Vietnamese,Chinese,Indonesian": 0.38, "Vietnamese,Chinese,English": 0.42, "Vietnamese,Spanish,Malay": 0.3333333333333333, "Vietnamese,Spanish,Indonesian": 0.3466666666666667, "Vietnamese,Spanish,English": 0.3466666666666667, "Vietnamese,Malay,Indonesian": 0.36666666666666664, "Vietnamese,Malay,English": 0.3333333333333333, "Vietnamese,Indonesian,English": 0.35333333333333333, "Chinese,Spanish,Malay": 0.34, "Chinese,Spanish,Indonesian": 0.36, "Chinese,Spanish,English": 0.36666666666666664, "Chinese,Malay,Indonesian": 0.36, "Chinese,Malay,English": 0.3466666666666667, "Chinese,Indonesian,English": 0.37333333333333335, "Spanish,Malay,Indonesian": 0.34, "Spanish,Malay,English": 0.29333333333333333, "Spanish,Indonesian,English": 0.30666666666666664, "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.26, "Filipino,Vietnamese,Chinese,Malay": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,English": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.23333333333333334, "Filipino,Chinese,Spanish,English": 0.25333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.24, "Filipino,Chinese,Malay,English": 0.22, "Filipino,Chinese,Indonesian,English": 0.24, "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Spanish,Malay,English": 0.2, "Filipino,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.26, "Vietnamese,Chinese,Spanish,Indonesian": 0.2733333333333333, "Vietnamese,Chinese,Spanish,English": 0.28, "Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, "Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.28, "Vietnamese,Spanish,Malay,Indonesian": 0.24, "Vietnamese,Spanish,Malay,English": 0.22, "Vietnamese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Malay,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.26, "Chinese,Spanish,Malay,English": 0.24, "Chinese,Spanish,Indonesian,English": 0.25333333333333335, "Chinese,Malay,Indonesian,English": 0.26666666666666666, "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.18, "Filipino,Chinese,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, "Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian,English": 0.22, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, "AC3_2": 0.4391168162111224, "AC3_3": 0.35800523336813417, "AC3_4": 0.29627093843513447, "AC3_5": 0.2484598592814412, "AC3_6": 0.21090024326859275, "AC3_7": 0.18184615380943492 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3092532467532468, "language_acc": { "Indonesian": 0.3068181818181818, "English": 0.2840909090909091, "Filipino": 0.2727272727272727, "Spanish": 0.3125, "Chinese": 0.32386363636363635, "Malay": 0.3522727272727273, "Vietnamese": 0.3125 }, "consistency_score_2": 0.4945887445887447, "consistency_score_3": 0.2849025974025974, "consistency_score_4": 0.18116883116883112, "consistency_score_5": 0.125, "consistency_score_6": 0.09253246753246754, "consistency_score_7": 0.07386363636363637, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4943181818181818, "Indonesian,Filipino": 0.4318181818181818, "Indonesian,Spanish": 0.4772727272727273, "Indonesian,Chinese": 0.5, "Indonesian,Malay": 0.5965909090909091, "Indonesian,Vietnamese": 0.5852272727272727, "English,Filipino": 0.4375, "English,Spanish": 0.5454545454545454, "English,Chinese": 0.5738636363636364, "English,Malay": 0.4772727272727273, "English,Vietnamese": 0.5, "Filipino,Spanish": 0.36363636363636365, "Filipino,Chinese": 0.3522727272727273, "Filipino,Malay": 0.4772727272727273, "Filipino,Vietnamese": 0.3977272727272727, "Spanish,Chinese": 0.5284090909090909, "Spanish,Malay": 0.4943181818181818, "Spanish,Vietnamese": 0.5568181818181818, "Chinese,Malay": 0.4431818181818182, "Chinese,Vietnamese": 0.5681818181818182, "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.23863636363636365, "Indonesian,English,Spanish": 0.29545454545454547, "Indonesian,English,Chinese": 0.3068181818181818, "Indonesian,English,Malay": 0.32386363636363635, "Indonesian,English,Vietnamese": 0.32386363636363635, "Indonesian,Filipino,Spanish": 0.21022727272727273, "Indonesian,Filipino,Chinese": 0.19886363636363635, "Indonesian,Filipino,Malay": 0.3068181818181818, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.2897727272727273, "Indonesian,Spanish,Malay": 0.32954545454545453, "Indonesian,Spanish,Vietnamese": 0.32386363636363635, "Indonesian,Chinese,Malay": 0.3068181818181818, "Indonesian,Chinese,Vietnamese": 0.3522727272727273, "Indonesian,Malay,Vietnamese": 0.4147727272727273, "English,Filipino,Spanish": 0.23863636363636365, "English,Filipino,Chinese": 0.24431818181818182, "English,Filipino,Malay": 0.26704545454545453, "English,Filipino,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese": 0.3522727272727273, "English,Spanish,Malay": 0.2840909090909091, "English,Spanish,Vietnamese": 0.32954545454545453, "English,Chinese,Malay": 0.26704545454545453, "English,Chinese,Vietnamese": 0.3465909090909091, "English,Malay,Vietnamese": 0.3125, "Filipino,Spanish,Chinese": 0.19318181818181818, "Filipino,Spanish,Malay": 0.22727272727272727, "Filipino,Spanish,Vietnamese": 0.20454545454545456, "Filipino,Chinese,Malay": 0.2215909090909091, "Filipino,Chinese,Vietnamese": 0.20454545454545456, "Filipino,Malay,Vietnamese": 0.2897727272727273, "Spanish,Chinese,Malay": 0.26704545454545453, "Spanish,Chinese,Vietnamese": 0.3465909090909091, "Spanish,Malay,Vietnamese": 0.3409090909090909, "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1590909090909091, "Indonesian,English,Filipino,Chinese": 0.14772727272727273, "Indonesian,English,Filipino,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Spanish,Malay": 0.22727272727272727, "Indonesian,English,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,English,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,English,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese,Malay": 0.19886363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Malay,Vietnamese": 0.25, "Indonesian,Chinese,Malay,Vietnamese": 0.24431818181818182, "English,Filipino,Spanish,Chinese": 0.1534090909090909, "English,Filipino,Spanish,Malay": 0.1590909090909091, "English,Filipino,Spanish,Vietnamese": 0.1534090909090909, "English,Filipino,Chinese,Malay": 0.14204545454545456, "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, "English,Filipino,Malay,Vietnamese": 0.17613636363636365, "English,Spanish,Chinese,Malay": 0.19318181818181818, "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, "English,Spanish,Malay,Vietnamese": 0.20454545454545456, "English,Chinese,Malay,Vietnamese": 0.21022727272727273, "Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Filipino,Spanish,Chinese,Vietnamese": 0.125, "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "English,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 } }, "AC3_2": 0.38055532480283305, "AC3_3": 0.2965789333118043, "AC3_4": 0.22848502039473678, "AC3_5": 0.17803738313657158, "AC3_6": 0.1424439196811943, "AC3_7": 0.11924595528474537 } }, "sg_eval": { "prompt_1": { "accuracy": 0.33980582524271846 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2571428571428571 } }, "us_eval": { "prompt_1": { "accuracy": 0.3925233644859813 } }, "ph_eval": { "prompt_1": { "accuracy": 0.32, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.3, "history": 0.4, "literature": 0.2, "politics": 0.4, "culture": 0.4, "film": 0.2, "law": 0.3, "geography": 0.4 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.2859232180639829 } }, "indommlu": { "prompt_1": { "accuracy": 0.33994258628746915, "category_acc": { "History": 0.29116465863453816, "Geography": 0.26326530612244897, "Lampungic": 0.32653061224489793, "Social science": 0.5158597662771286, "Balinese": 0.2781316348195329, "Makassarese": 0.27956989247311825, "Banjarese": 0.2361111111111111, "Chemistry": 0.23503649635036497, "Biology": 0.3029585798816568, "Science": 0.3735810113519092, "Christian religion": 0.3880597014925373, "Art": 0.39767054908485855, "Islam religion": 0.4039829302987198, "Hindu religion": 0.35333333333333333, "Madurese": 0.27796610169491526, "Sport": 0.34459459459459457, "Indonesian language": 0.3788916562889166, "Physics": 0.2828282828282828, "Minangkabau culture": 0.32663316582914576, "Dayak language": 0.22935779816513763, "Sociology": 0.33669354838709675, "Economy": 0.28688524590163933, "Sundanese": 0.3025064822817632, "Javanese": 0.2842741935483871, "Civic education": 0.4177396280400572 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.3824192602359019 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.3128439608609861 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.2169986161156318 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.38545186546036936 } }, "mmlu": { "prompt_1": { "accuracy": 0.3547257876312719 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.3407937075437969, "category_acc": { "high_school_european_history": 0.4146341463414634, "business_ethics": 0.32323232323232326, "clinical_knowledge": 0.4053030303030303, "medical_genetics": 0.3838383838383838, "high_school_us_history": 0.4433497536945813, "high_school_physics": 0.38, "high_school_world_history": 0.3601694915254237, "virology": 0.2727272727272727, "high_school_microeconomics": 0.25316455696202533, "econometrics": 0.25663716814159293, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.35275080906148865, "abstract_algebra": 0.20202020202020202, "professional_accounting": 0.28825622775800713, "philosophy": 0.36129032258064514, "professional_medicine": 0.35793357933579334, "nutrition": 0.38688524590163936, "global_facts": 0.23232323232323232, "machine_learning": 0.25225225225225223, "security_studies": 0.3483606557377049, "public_relations": 0.28440366972477066, "professional_psychology": 0.353518821603928, "prehistory": 0.3653250773993808, "anatomy": 0.3582089552238806, "human_sexuality": 0.43846153846153846, "college_medicine": 0.3430232558139535, "high_school_government_and_politics": 0.4635416666666667, "college_chemistry": 0.3434343434343434, "logical_fallacies": 0.35802469135802467, "high_school_geography": 0.4720812182741117, "elementary_mathematics": 0.2864721485411141, "human_aging": 0.3108108108108108, "college_mathematics": 0.29292929292929293, "high_school_psychology": 0.375, "formal_logic": 0.264, "high_school_statistics": 0.3116279069767442, "international_law": 0.45, "high_school_mathematics": 0.23048327137546468, "high_school_computer_science": 0.3939393939393939, "conceptual_physics": 0.32905982905982906, "miscellaneous": 0.4143222506393862, "high_school_chemistry": 0.3118811881188119, "marketing": 0.4034334763948498, "professional_law": 0.2759295499021526, "management": 0.39215686274509803, "college_physics": 0.31683168316831684, "jurisprudence": 0.3177570093457944, "world_religions": 0.4823529411764706, "sociology": 0.43, "us_foreign_policy": 0.5151515151515151, "high_school_macroeconomics": 0.29562982005141386, "computer_security": 0.3838383838383838, "moral_scenarios": 0.24496644295302014, "moral_disputes": 0.37971014492753624, "electrical_engineering": 0.3333333333333333, "astronomy": 0.40397350993377484, "college_biology": 0.3916083916083916 } } }, "c_eval": { "prompt_1": { "accuracy": 0.3150074294205052 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.30884184308841844, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.08333333333333333, "computer_architecture": 0.19230769230769232, "college_programming": 0.19047619047619047, "college_physics": 0.4166666666666667, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.125, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.25, "high_school_chemistry": 0.25, "high_school_biology": 0.125, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.23333333333333334, "business_administration": 0.2894736842105263, "marxism": 0.3333333333333333, "mao_zedong_thought": 0.3793103448275862, "education_science": 0.2647058823529412, "teacher_qualification": 0.30612244897959184, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.375, "logic": 0.25925925925925924, "law": 0.27586206896551724, "chinese_language_and_literature": 0.25, "art_studies": 0.2894736842105263, "professional_tour_guide": 0.3235294117647059, "legal_professional": 0.5357142857142857, "high_school_chinese": 0.4583333333333333, "high_school_history": 0.36, "middle_school_history": 0.3333333333333333, "civil_servant": 0.38461538461538464, "sports_science": 0.375, "plant_protection": 0.25925925925925924, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.21568627450980393, "accountant": 0.4074074074074074, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.3055555555555556, "tax_accountant": 0.2037037037037037, "physician": 0.3148148148148148 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.2616487455197133 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.28708340528406145, "category_acc": { "agronomy": 0.27218934911242604, "anatomy": 0.25, "ancient_chinese": 0.29878048780487804, "arts": 0.3375, "astronomy": 0.28484848484848485, "business_ethics": 0.23444976076555024, "chinese_civil_service_exam": 0.25, "chinese_driving_rule": 0.3282442748091603, "chinese_food_culture": 0.3161764705882353, "chinese_foreign_policy": 0.29906542056074764, "chinese_history": 0.29102167182662536, "chinese_literature": 0.29411764705882354, "chinese_teacher_qualification": 0.30726256983240224, "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.27358490566037735, "college_education": 0.32710280373831774, "college_engineering_hydrology": 0.22641509433962265, "college_law": 0.23148148148148148, "college_mathematics": 0.19047619047619047, "college_medical_statistics": 0.3490566037735849, "college_medicine": 0.2783882783882784, "computer_science": 0.25980392156862747, "computer_security": 0.3157894736842105, "conceptual_physics": 0.30612244897959184, "construction_project_management": 0.2949640287769784, "economics": 0.2893081761006289, "education": 0.31901840490797545, "electrical_engineering": 0.32558139534883723, "elementary_chinese": 0.2896825396825397, "elementary_commonsense": 0.24242424242424243, "elementary_information_and_technology": 0.33613445378151263, "elementary_mathematics": 0.2391304347826087, "ethnology": 0.3333333333333333, "food_science": 0.2937062937062937, "genetics": 0.2556818181818182, "global_facts": 0.35570469798657717, "high_school_biology": 0.28994082840236685, "high_school_chemistry": 0.24242424242424243, "high_school_geography": 0.2627118644067797, "high_school_mathematics": 0.24390243902439024, "high_school_physics": 0.20909090909090908, "high_school_politics": 0.3146853146853147, "human_sexuality": 0.29365079365079366, "international_law": 0.2594594594594595, "journalism": 0.29651162790697677, "jurisprudence": 0.2846715328467153, "legal_and_moral_basis": 0.40654205607476634, "logical": 0.2682926829268293, "machine_learning": 0.27049180327868855, "management": 0.29523809523809524, "marketing": 0.29444444444444445, "marxist_theory": 0.32275132275132273, "modern_chinese": 0.25862068965517243, "nutrition": 0.2620689655172414, "philosophy": 0.3142857142857143, "professional_accounting": 0.2742857142857143, "professional_law": 0.2890995260663507, "professional_medicine": 0.24202127659574468, "professional_psychology": 0.31896551724137934, "public_relations": 0.2471264367816092, "security_study": 0.37037037037037035, "sociology": 0.2610619469026549, "sports_science": 0.3333333333333333, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.3136094674556213, "world_history": 0.30434782608695654, "world_religions": 0.3 } } }, "zbench": { "prompt_1": { "accuracy": 0.2727272727272727 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.18636363636363637 } }, "ocnli": { "prompt_1": { "accuracy": 0.3288135593220339 } }, "c3": { "prompt_1": { "accuracy": 0.4143605086013463 } }, "dream": { "prompt_1": { "accuracy": 0.5227829495345419 } }, "samsum": { "prompt_1": { "rouge1": 0.41687717824020265, "rouge2": 0.17505033429566638, "rougeL": 0.33363971351255944, "avg_rouge": 0.3085224086828095 } }, "dialogsum": { "prompt_1": { "rouge1": 0.39976991490542363, "rouge2": 0.15292232665909905, "rougeL": 0.3215121347124994, "avg_rouge": 0.29140145875900736 } }, "sst2": { "prompt_1": { "accuracy": 0.5630733944954128 } }, "cola": { "prompt_1": { "accuracy": 0.5321188878235859 } }, "qqp": { "prompt_1": { "accuracy": 0.4955 } }, "mnli": { "prompt_1": { "accuracy": 0.4 } }, "qnli": { "prompt_1": { "accuracy": 0.7 } }, "wnli": { "prompt_1": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.4 } }, "mrpc": { "prompt_1": { "accuracy": 0.6 } } } }, "sailor_7b": { "model_size": "7B", "model_link": "https://huggingface.co/sail/Sailor-7B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.7960084033613446, "language_acc": { "Spanish": 0.7798319327731092, "English": 0.8134453781512605, "Chinese": 0.7974789915966387, "Vietnamese": 0.7932773109243697 }, "consistency_score_2": 0.8275910364145659, "consistency_score_3": 0.751890756302521, "consistency_score_4": 0.704201680672269, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8344537815126051, "Spanish,Chinese": 0.8016806722689076, "Spanish,Vietnamese": 0.8252100840336134, "English,Chinese": 0.8336134453781513, "English,Vietnamese": 0.8529411764705882, "Chinese,Vietnamese": 0.8176470588235294 }, "3_combine": { "Spanish,English,Chinese": 0.746218487394958, "Spanish,English,Vietnamese": 0.7647058823529411, "Spanish,Chinese,Vietnamese": 0.7336134453781512, "English,Chinese,Vietnamese": 0.7630252100840336 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.704201680672269 } }, "AC3_2": 0.8114925434845776, "AC3_3": 0.7733208673849625, "AC3_4": 0.747295943954452 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.5219047619047619, "language_acc": { "Filipino": 0.36, "Vietnamese": 0.5266666666666666, "Chinese": 0.5466666666666666, "Spanish": 0.48, "Malay": 0.5133333333333333, "Indonesian": 0.5933333333333334, "English": 0.6333333333333333 }, "consistency_score_2": 0.6107936507936507, "consistency_score_3": 0.4615238095238095, "consistency_score_4": 0.3739047619047619, "consistency_score_5": 0.313015873015873, "consistency_score_6": 0.2657142857142857, "consistency_score_7": 0.22666666666666666, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4866666666666667, "Filipino,Chinese": 0.4866666666666667, "Filipino,Spanish": 0.4666666666666667, "Filipino,Malay": 0.4866666666666667, "Filipino,Indonesian": 0.5066666666666667, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.6333333333333333, "Vietnamese,Spanish": 0.5933333333333334, "Vietnamese,Malay": 0.6533333333333333, "Vietnamese,Indonesian": 0.6333333333333333, "Vietnamese,English": 0.6866666666666666, "Chinese,Spanish": 0.6466666666666666, "Chinese,Malay": 0.6733333333333333, "Chinese,Indonesian": 0.6866666666666666, "Chinese,English": 0.7333333333333333, "Spanish,Malay": 0.6133333333333333, "Spanish,Indonesian": 0.6333333333333333, "Spanish,English": 0.6333333333333333, "Malay,Indonesian": 0.6666666666666666, "Malay,English": 0.7133333333333334, "Indonesian,English": 0.7266666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.34, "Filipino,Vietnamese,Spanish": 0.34, "Filipino,Vietnamese,Malay": 0.36666666666666664, "Filipino,Vietnamese,Indonesian": 0.36, "Filipino,Vietnamese,English": 0.36666666666666664, "Filipino,Chinese,Spanish": 0.36666666666666664, "Filipino,Chinese,Malay": 0.38, "Filipino,Chinese,Indonesian": 0.3933333333333333, "Filipino,Chinese,English": 0.38666666666666666, "Filipino,Spanish,Malay": 0.35333333333333333, "Filipino,Spanish,Indonesian": 0.38, "Filipino,Spanish,English": 0.36, "Filipino,Malay,Indonesian": 0.3933333333333333, "Filipino,Malay,English": 0.38666666666666666, "Filipino,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Spanish": 0.4866666666666667, "Vietnamese,Chinese,Malay": 0.5133333333333333, "Vietnamese,Chinese,Indonesian": 0.5, "Vietnamese,Chinese,English": 0.5533333333333333, "Vietnamese,Spanish,Malay": 0.4866666666666667, "Vietnamese,Spanish,Indonesian": 0.48, "Vietnamese,Spanish,English": 0.5, "Vietnamese,Malay,Indonesian": 0.5066666666666667, "Vietnamese,Malay,English": 0.56, "Vietnamese,Indonesian,English": 0.5466666666666666, "Chinese,Spanish,Malay": 0.5133333333333333, "Chinese,Spanish,Indonesian": 0.5133333333333333, "Chinese,Spanish,English": 0.5533333333333333, "Chinese,Malay,Indonesian": 0.54, "Chinese,Malay,English": 0.6, "Chinese,Indonesian,English": 0.6066666666666667, "Spanish,Malay,Indonesian": 0.4866666666666667, "Spanish,Malay,English": 0.5266666666666666, "Spanish,Indonesian,English": 0.5333333333333333, "Malay,Indonesian,English": 0.58 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.3, "Filipino,Vietnamese,Chinese,Indonesian": 0.3, "Filipino,Vietnamese,Chinese,English": 0.30666666666666664, "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Spanish,English": 0.3, "Filipino,Vietnamese,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Malay,English": 0.31333333333333335, "Filipino,Vietnamese,Indonesian,English": 0.31333333333333335, "Filipino,Chinese,Spanish,Malay": 0.31333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Chinese,Spanish,English": 0.3333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.3466666666666667, "Filipino,Chinese,Malay,English": 0.3466666666666667, "Filipino,Chinese,Indonesian,English": 0.35333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, "Filipino,Spanish,Malay,English": 0.30666666666666664, "Filipino,Spanish,Indonesian,English": 0.32666666666666666, "Filipino,Malay,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.43333333333333335, "Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.43333333333333335, "Vietnamese,Chinese,Malay,English": 0.4866666666666667, "Vietnamese,Chinese,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Spanish,Malay,English": 0.44666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.44, "Vietnamese,Malay,Indonesian,English": 0.47333333333333333, "Chinese,Spanish,Malay,Indonesian": 0.43333333333333335, "Chinese,Spanish,Malay,English": 0.4866666666666667, "Chinese,Spanish,Indonesian,English": 0.48, "Chinese,Malay,Indonesian,English": 0.5, "Spanish,Malay,Indonesian,English": 0.44 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.28, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.28, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.26, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.29333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.30666666666666664, "Filipino,Chinese,Malay,Indonesian,English": 0.31333333333333335, "Filipino,Spanish,Malay,Indonesian,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Malay,English": 0.42, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.41333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.38666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.41333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 } }, "AC3_2": 0.5628614135740533, "AC3_3": 0.48986063837561483, "AC3_4": 0.4356789485454739, "AC3_5": 0.39132934994407675, "AC3_6": 0.35214372080517775, "AC3_7": 0.31606446136575045 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.42045454545454547, "language_acc": { "Indonesian": 0.48295454545454547, "English": 0.45454545454545453, "Filipino": 0.3068181818181818, "Spanish": 0.42613636363636365, "Chinese": 0.42045454545454547, "Malay": 0.4375, "Vietnamese": 0.4147727272727273 }, "consistency_score_2": 0.49918831168831174, "consistency_score_3": 0.3222402597402598, "consistency_score_4": 0.23490259740259742, "consistency_score_5": 0.18290043290043292, "consistency_score_6": 0.14853896103896105, "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5681818181818182, "Indonesian,Filipino": 0.3977272727272727, "Indonesian,Spanish": 0.5056818181818182, "Indonesian,Chinese": 0.4715909090909091, "Indonesian,Malay": 0.6534090909090909, "Indonesian,Vietnamese": 0.5625, "English,Filipino": 0.4659090909090909, "English,Spanish": 0.5170454545454546, "English,Chinese": 0.5909090909090909, "English,Malay": 0.5738636363636364, "English,Vietnamese": 0.5909090909090909, "Filipino,Spanish": 0.3352272727272727, "Filipino,Chinese": 0.48863636363636365, "Filipino,Malay": 0.38636363636363635, "Filipino,Vietnamese": 0.3806818181818182, "Spanish,Chinese": 0.4431818181818182, "Spanish,Malay": 0.48295454545454547, "Spanish,Vietnamese": 0.5625, "Chinese,Malay": 0.4715909090909091, "Chinese,Vietnamese": 0.5056818181818182, "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.2897727272727273, "Indonesian,English,Spanish": 0.3693181818181818, "Indonesian,English,Chinese": 0.3693181818181818, "Indonesian,English,Malay": 0.4375, "Indonesian,English,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Spanish": 0.22727272727272727, "Indonesian,Filipino,Chinese": 0.26136363636363635, "Indonesian,Filipino,Malay": 0.29545454545454547, "Indonesian,Filipino,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Chinese": 0.2897727272727273, "Indonesian,Spanish,Malay": 0.3693181818181818, "Indonesian,Spanish,Vietnamese": 0.3693181818181818, "Indonesian,Chinese,Malay": 0.3806818181818182, "Indonesian,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,Malay,Vietnamese": 0.42045454545454547, "English,Filipino,Spanish": 0.2727272727272727, "English,Filipino,Chinese": 0.3522727272727273, "English,Filipino,Malay": 0.2840909090909091, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.3352272727272727, "English,Spanish,Malay": 0.36363636363636365, "English,Spanish,Vietnamese": 0.3806818181818182, "English,Chinese,Malay": 0.3806818181818182, "English,Chinese,Vietnamese": 0.3977272727272727, "English,Malay,Vietnamese": 0.4034090909090909, "Filipino,Spanish,Chinese": 0.23295454545454544, "Filipino,Spanish,Malay": 0.21022727272727273, "Filipino,Spanish,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay": 0.26704545454545453, "Filipino,Chinese,Vietnamese": 0.2727272727272727, "Filipino,Malay,Vietnamese": 0.23863636363636365, "Spanish,Chinese,Malay": 0.2897727272727273, "Spanish,Chinese,Vietnamese": 0.3068181818181818, "Spanish,Malay,Vietnamese": 0.3409090909090909, "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.20454545454545456, "Indonesian,English,Filipino,Chinese": 0.23863636363636365, "Indonesian,English,Filipino,Malay": 0.23295454545454544, "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Chinese": 0.2556818181818182, "Indonesian,English,Spanish,Malay": 0.3068181818181818, "Indonesian,English,Spanish,Vietnamese": 0.29545454545454547, "Indonesian,English,Chinese,Malay": 0.3125, "Indonesian,English,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,English,Malay,Vietnamese": 0.3352272727272727, "Indonesian,Filipino,Spanish,Chinese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Chinese,Malay": 0.2159090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese,Malay": 0.25, "Indonesian,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Spanish,Malay,Vietnamese": 0.2897727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish,Chinese": 0.2159090909090909, "English,Filipino,Spanish,Malay": 0.19318181818181818, "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, "English,Filipino,Chinese,Malay": 0.23863636363636365, "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, "English,Filipino,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay": 0.26136363636363635, "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, "English,Spanish,Malay,Vietnamese": 0.2840909090909091, "English,Chinese,Malay,Vietnamese": 0.29545454545454547, "Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.1875, "Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, "Indonesian,English,Filipino,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.25, "Indonesian,English,Chinese,Malay,Vietnamese": 0.25, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, "AC3_2": 0.45645109519228255, "AC3_3": 0.3648534525092451, "AC3_4": 0.301410813564475, "AC3_5": 0.25491235217952013, "AC3_6": 0.21952405650402038, "AC3_7": 0.1927083332980035 } }, "sg_eval": { "prompt_1": { "accuracy": 0.5631067961165048 } }, "cn_eval": { "prompt_1": { "accuracy": 0.6 } }, "us_eval": { "prompt_1": { "accuracy": 0.5887850467289719 } }, "ph_eval": { "prompt_1": { "accuracy": 0.37, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.5, "culture": 0.3, "film": 0.2, "law": 0.7, "geography": 0.5 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.32178565305954265 } }, "indommlu": { "prompt_1": { "accuracy": 0.490219640830496, "category_acc": { "History": 0.5321285140562249, "Geography": 0.44285714285714284, "Lampungic": 0.3741496598639456, "Social science": 0.7378964941569283, "Balinese": 0.3248407643312102, "Makassarese": 0.34946236559139787, "Banjarese": 0.4166666666666667, "Chemistry": 0.28175182481751826, "Biology": 0.4437869822485207, "Science": 0.6140350877192983, "Christian religion": 0.6218905472636815, "Art": 0.5973377703826955, "Islam religion": 0.6244665718349929, "Hindu religion": 0.48, "Madurese": 0.26440677966101694, "Sport": 0.5675675675675675, "Indonesian language": 0.547011207970112, "Physics": 0.3696969696969697, "Minangkabau culture": 0.3869346733668342, "Dayak language": 0.3302752293577982, "Sociology": 0.45161290322580644, "Economy": 0.4487704918032787, "Sundanese": 0.3975799481417459, "Javanese": 0.3719758064516129, "Civic education": 0.630901287553648 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.4081953488837232 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.33776063614449015 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.2477760479226549 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.4184670269252103 } }, "mmlu": { "prompt_1": { "accuracy": 0.4830805134189031 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.5003217733285663, "category_acc": { "high_school_european_history": 0.6219512195121951, "business_ethics": 0.494949494949495, "clinical_knowledge": 0.5795454545454546, "medical_genetics": 0.5555555555555556, "high_school_us_history": 0.6699507389162561, "high_school_physics": 0.32, "high_school_world_history": 0.635593220338983, "virology": 0.43636363636363634, "high_school_microeconomics": 0.48945147679324896, "econometrics": 0.3274336283185841, "college_computer_science": 0.41414141414141414, "high_school_biology": 0.6245954692556634, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.35587188612099646, "philosophy": 0.5774193548387097, "professional_medicine": 0.4907749077490775, "nutrition": 0.5770491803278689, "global_facts": 0.3333333333333333, "machine_learning": 0.44144144144144143, "security_studies": 0.569672131147541, "public_relations": 0.5504587155963303, "professional_psychology": 0.4746317512274959, "prehistory": 0.5882352941176471, "anatomy": 0.5522388059701493, "human_sexuality": 0.5923076923076923, "college_medicine": 0.46511627906976744, "high_school_government_and_politics": 0.7135416666666666, "college_chemistry": 0.46464646464646464, "logical_fallacies": 0.5864197530864198, "high_school_geography": 0.6649746192893401, "elementary_mathematics": 0.3209549071618037, "human_aging": 0.5990990990990991, "college_mathematics": 0.36363636363636365, "high_school_psychology": 0.6709558823529411, "formal_logic": 0.336, "high_school_statistics": 0.4, "international_law": 0.6583333333333333, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.5151515151515151, "conceptual_physics": 0.49145299145299143, "miscellaneous": 0.7161125319693095, "high_school_chemistry": 0.4306930693069307, "marketing": 0.776824034334764, "professional_law": 0.3457273320287019, "management": 0.6078431372549019, "college_physics": 0.27722772277227725, "jurisprudence": 0.6074766355140186, "world_religions": 0.6764705882352942, "sociology": 0.655, "us_foreign_policy": 0.797979797979798, "high_school_macroeconomics": 0.4832904884318766, "computer_security": 0.6666666666666666, "moral_scenarios": 0.24049217002237136, "moral_disputes": 0.5478260869565217, "electrical_engineering": 0.4722222222222222, "astronomy": 0.5695364238410596, "college_biology": 0.5524475524475524 } } }, "c_eval": { "prompt_1": { "accuracy": 0.512630014858841 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.5336239103362391, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.5833333333333334, "computer_architecture": 0.4230769230769231, "college_programming": 0.6190476190476191, "college_physics": 0.4583333333333333, "college_chemistry": 0.4827586206896552, "advanced_mathematics": 0.375, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.38095238095238093, "metrology_engineer": 0.5862068965517241, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.5, "high_school_chemistry": 0.625, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.4166666666666667, "middle_school_biology": 0.8461538461538461, "middle_school_physics": 0.75, "middle_school_chemistry": 0.96, "veterinary_medicine": 0.5, "college_economics": 0.43333333333333335, "business_administration": 0.4473684210526316, "marxism": 0.6666666666666666, "mao_zedong_thought": 0.8275862068965517, "education_science": 0.6470588235294118, "teacher_qualification": 0.7551020408163265, "high_school_politics": 0.7916666666666666, "high_school_geography": 0.7083333333333334, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.7058823529411765, "modern_chinese_history": 0.6428571428571429, "ideological_and_moral_cultivation": 0.7916666666666666, "logic": 0.4444444444444444, "law": 0.27586206896551724, "chinese_language_and_literature": 0.5714285714285714, "art_studies": 0.5526315789473685, "professional_tour_guide": 0.5588235294117647, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.5, "high_school_history": 0.8, "middle_school_history": 0.6296296296296297, "civil_servant": 0.34615384615384615, "sports_science": 0.4583333333333333, "plant_protection": 0.5925925925925926, "basic_medicine": 0.5, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.5882352941176471, "accountant": 0.4444444444444444, "fire_engineer": 0.4166666666666667, "environmental_impact_assessment_engineer": 0.5, "tax_accountant": 0.46296296296296297, "physician": 0.5555555555555556 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.5770609318996416 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.5230530132964946, "category_acc": { "agronomy": 0.47928994082840237, "anatomy": 0.33783783783783783, "ancient_chinese": 0.36585365853658536, "arts": 0.76875, "astronomy": 0.3333333333333333, "business_ethics": 0.507177033492823, "chinese_civil_service_exam": 0.4625, "chinese_driving_rule": 0.6412213740458015, "chinese_food_culture": 0.5294117647058824, "chinese_foreign_policy": 0.6261682242990654, "chinese_history": 0.6222910216718266, "chinese_literature": 0.4019607843137255, "chinese_teacher_qualification": 0.664804469273743, "clinical_knowledge": 0.4430379746835443, "college_actuarial_science": 0.2358490566037736, "college_education": 0.7102803738317757, "college_engineering_hydrology": 0.4339622641509434, "college_law": 0.46296296296296297, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.37735849056603776, "college_medicine": 0.4835164835164835, "computer_science": 0.5343137254901961, "computer_security": 0.6257309941520468, "conceptual_physics": 0.5578231292517006, "construction_project_management": 0.4172661870503597, "economics": 0.5660377358490566, "education": 0.6196319018404908, "electrical_engineering": 0.5, "elementary_chinese": 0.5158730158730159, "elementary_commonsense": 0.601010101010101, "elementary_information_and_technology": 0.7184873949579832, "elementary_mathematics": 0.3217391304347826, "ethnology": 0.5111111111111111, "food_science": 0.5384615384615384, "genetics": 0.4147727272727273, "global_facts": 0.5838926174496645, "high_school_biology": 0.5384615384615384, "high_school_chemistry": 0.4696969696969697, "high_school_geography": 0.6271186440677966, "high_school_mathematics": 0.31097560975609756, "high_school_physics": 0.4090909090909091, "high_school_politics": 0.5594405594405595, "human_sexuality": 0.5476190476190477, "international_law": 0.4540540540540541, "journalism": 0.5697674418604651, "jurisprudence": 0.5523114355231143, "legal_and_moral_basis": 0.8738317757009346, "logical": 0.4146341463414634, "machine_learning": 0.4098360655737705, "management": 0.6190476190476191, "marketing": 0.5444444444444444, "marxist_theory": 0.7195767195767195, "modern_chinese": 0.4051724137931034, "nutrition": 0.5655172413793104, "philosophy": 0.6, "professional_accounting": 0.5828571428571429, "professional_law": 0.44075829383886256, "professional_medicine": 0.39361702127659576, "professional_psychology": 0.5775862068965517, "public_relations": 0.5459770114942529, "security_study": 0.5851851851851851, "sociology": 0.5221238938053098, "sports_science": 0.5212121212121212, "traditional_chinese_medicine": 0.4540540540540541, "virology": 0.5325443786982249, "world_history": 0.639751552795031, "world_religions": 0.575 } } }, "zbench": { "prompt_1": { "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.26590909090909093 } }, "ocnli": { "prompt_1": { "accuracy": 0.32372881355932204 } }, "c3": { "prompt_1": { "accuracy": 0.7550486163051608 } }, "dream": { "prompt_1": { "accuracy": 0.7437530622243999 } }, "samsum": { "prompt_1": { "rouge1": 0.45036232390257586, "rouge2": 0.20575253780038455, "rougeL": 0.3643735012321646, "avg_rouge": 0.34016278764504165 } }, "dialogsum": { "prompt_1": { "rouge1": 0.4231691817172813, "rouge2": 0.17154835835598875, "rougeL": 0.35000262687479355, "avg_rouge": 0.3149067223160212 } }, "sst2": { "prompt_1": { "accuracy": 0.7821100917431193 } }, "cola": { "prompt_1": { "accuracy": 0.5944391179290508 } }, "qqp": { "prompt_1": { "accuracy": 0.509 } }, "mnli": { "prompt_1": { "accuracy": 0.2 } }, "qnli": { "prompt_1": { "accuracy": 0.6 } }, "wnli": { "prompt_1": { "accuracy": 0.6 } }, "rte": { "prompt_1": { "accuracy": 0.5 } }, "mrpc": { "prompt_1": { "accuracy": 0.5 } } } }, "sailor_0_5b_chat": { "model_size": "0.5B", "model_link": "https://huggingface.co/sail/Sailor-0.5B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.3468487394957983, "language_acc": { "Spanish": 0.33361344537815124, "English": 0.380672268907563, "Chinese": 0.32857142857142857, "Vietnamese": 0.3445378151260504 }, "consistency_score_2": 0.3995798319327731, "consistency_score_3": 0.1953781512605042, "consistency_score_4": 0.10336134453781512, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.4504201680672269, "Spanish,Chinese": 0.36554621848739494, "Spanish,Vietnamese": 0.47058823529411764, "English,Chinese": 0.37815126050420167, "English,Vietnamese": 0.4008403361344538, "Chinese,Vietnamese": 0.3319327731092437 }, "3_combine": { "Spanish,English,Chinese": 0.1915966386554622, "Spanish,English,Vietnamese": 0.2453781512605042, "Spanish,Chinese,Vietnamese": 0.18067226890756302, "English,Chinese,Vietnamese": 0.1638655462184874 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.10336134453781512 } }, "AC3_2": 0.3713517041557144, "AC3_3": 0.249956859877259, "AC3_4": 0.1592623236528496 }, "prompt_2": { "overall_acc": 0.3340336134453782, "language_acc": { "Spanish": 0.33865546218487397, "English": 0.34789915966386553, "Chinese": 0.3100840336134454, "Vietnamese": 0.33949579831932775 }, "consistency_score_2": 0.47142857142857136, "consistency_score_3": 0.2819327731092437, "consistency_score_4": 0.19327731092436976, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.5042016806722689, "Spanish,Chinese": 0.4764705882352941, "Spanish,Vietnamese": 0.4823529411764706, "English,Chinese": 0.5134453781512605, "English,Vietnamese": 0.46050420168067224, "Chinese,Vietnamese": 0.3915966386554622 }, "3_combine": { "Spanish,English,Chinese": 0.31596638655462184, "Spanish,English,Vietnamese": 0.292436974789916, "Spanish,Chinese,Vietnamese": 0.25630252100840334, "English,Chinese,Vietnamese": 0.2630252100840336 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.19327731092436976 } }, "AC3_2": 0.391012743076874, "AC3_3": 0.3057797470470136, "AC3_4": 0.24486926239957693 }, "prompt_3": { "overall_acc": 0.3401260504201681, "language_acc": { "Spanish": 0.3445378151260504, "English": 0.3487394957983193, "Chinese": 0.32941176470588235, "Vietnamese": 0.3378151260504202 }, "consistency_score_2": 0.4343137254901961, "consistency_score_3": 0.2346638655462185, "consistency_score_4": 0.13865546218487396, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.47478991596638653, "Spanish,Chinese": 0.40252100840336136, "Spanish,Vietnamese": 0.4857142857142857, "English,Chinese": 0.45630252100840335, "English,Vietnamese": 0.43781512605042017, "Chinese,Vietnamese": 0.3487394957983193 }, "3_combine": { "Spanish,English,Chinese": 0.24873949579831933, "Spanish,English,Vietnamese": 0.27563025210084036, "Spanish,Chinese,Vietnamese": 0.20840336134453782, "English,Chinese,Vietnamese": 0.20588235294117646 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.13865546218487396 } }, "AC3_2": 0.38149231656270854, "AC3_3": 0.2777198817640574, "AC3_4": 0.19700148594157488 }, "prompt_4": { "overall_acc": 0.3191176470588235, "language_acc": { "Spanish": 0.3226890756302521, "English": 0.3176470588235294, "Chinese": 0.32521008403361346, "Vietnamese": 0.31092436974789917 }, "consistency_score_2": 0.4189075630252101, "consistency_score_3": 0.2130252100840336, "consistency_score_4": 0.11932773109243698, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.4588235294117647, "Spanish,Chinese": 0.4142857142857143, "Spanish,Vietnamese": 0.44285714285714284, "English,Chinese": 0.46218487394957986, "English,Vietnamese": 0.3739495798319328, "Chinese,Vietnamese": 0.36134453781512604 }, "3_combine": { "Spanish,English,Chinese": 0.25126050420168067, "Spanish,English,Vietnamese": 0.21428571428571427, "Spanish,Chinese,Vietnamese": 0.19663865546218487, "English,Chinese,Vietnamese": 0.1899159663865546 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.11932773109243698 } }, "AC3_2": 0.3622662044685879, "AC3_3": 0.2554956921096242, "AC3_4": 0.1737027537020321 }, "prompt_5": { "overall_acc": 0.32941176470588235, "language_acc": { "Spanish": 0.334453781512605, "English": 0.32016806722689073, "Chinese": 0.3411764705882353, "Vietnamese": 0.32184873949579834 }, "consistency_score_2": 0.42198879551820734, "consistency_score_3": 0.2161764705882353, "consistency_score_4": 0.126890756302521, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.4226890756302521, "Spanish,Chinese": 0.44873949579831934, "Spanish,Vietnamese": 0.4647058823529412, "English,Chinese": 0.43025210084033616, "English,Vietnamese": 0.38823529411764707, "Chinese,Vietnamese": 0.3773109243697479 }, "3_combine": { "Spanish,English,Chinese": 0.22605042016806723, "Spanish,English,Vietnamese": 0.22521008403361345, "Spanish,Chinese,Vietnamese": 0.2235294117647059, "English,Chinese,Vietnamese": 0.1899159663865546 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.126890756302521 } }, "AC3_2": 0.36999725887296725, "AC3_3": 0.2610432851907775, "AC3_4": 0.18320875307435475 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.25047619047619046, "language_acc": { "Filipino": 0.26666666666666666, "Vietnamese": 0.25333333333333335, "Chinese": 0.26666666666666666, "Spanish": 0.23333333333333334, "Malay": 0.26, "Indonesian": 0.21333333333333335, "English": 0.26 }, "consistency_score_2": 0.5441269841269841, "consistency_score_3": 0.3451428571428572, "consistency_score_4": 0.23638095238095239, "consistency_score_5": 0.17079365079365078, "consistency_score_6": 0.1285714285714286, "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5933333333333334, "Filipino,Chinese": 0.5466666666666666, "Filipino,Spanish": 0.58, "Filipino,Malay": 0.68, "Filipino,Indonesian": 0.6733333333333333, "Filipino,English": 0.5066666666666667, "Vietnamese,Chinese": 0.5, "Vietnamese,Spanish": 0.5133333333333333, "Vietnamese,Malay": 0.54, "Vietnamese,Indonesian": 0.5866666666666667, "Vietnamese,English": 0.43333333333333335, "Chinese,Spanish": 0.47333333333333333, "Chinese,Malay": 0.5533333333333333, "Chinese,Indonesian": 0.5266666666666666, "Chinese,English": 0.5, "Spanish,Malay": 0.5, "Spanish,Indonesian": 0.5733333333333334, "Spanish,English": 0.52, "Malay,Indonesian": 0.76, "Malay,English": 0.44, "Indonesian,English": 0.4266666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.35333333333333333, "Filipino,Vietnamese,Spanish": 0.36666666666666664, "Filipino,Vietnamese,Malay": 0.42, "Filipino,Vietnamese,Indonesian": 0.44, "Filipino,Vietnamese,English": 0.3, "Filipino,Chinese,Spanish": 0.3333333333333333, "Filipino,Chinese,Malay": 0.4066666666666667, "Filipino,Chinese,Indonesian": 0.38666666666666666, "Filipino,Chinese,English": 0.32, "Filipino,Spanish,Malay": 0.3933333333333333, "Filipino,Spanish,Indonesian": 0.42, "Filipino,Spanish,English": 0.3333333333333333, "Filipino,Malay,Indonesian": 0.5666666666666667, "Filipino,Malay,English": 0.3333333333333333, "Filipino,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish": 0.28, "Vietnamese,Chinese,Malay": 0.3466666666666667, "Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,English": 0.24666666666666667, "Vietnamese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Spanish,Indonesian": 0.36, "Vietnamese,Spanish,English": 0.2733333333333333, "Vietnamese,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Malay,English": 0.24, "Vietnamese,Indonesian,English": 0.26, "Chinese,Spanish,Malay": 0.30666666666666664, "Chinese,Spanish,Indonesian": 0.32, "Chinese,Spanish,English": 0.2866666666666667, "Chinese,Malay,Indonesian": 0.43333333333333335, "Chinese,Malay,English": 0.29333333333333333, "Chinese,Indonesian,English": 0.2733333333333333, "Spanish,Malay,Indonesian": 0.4266666666666667, "Spanish,Malay,English": 0.2733333333333333, "Spanish,Indonesian,English": 0.2866666666666667, "Malay,Indonesian,English": 0.3466666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.28, "Filipino,Vietnamese,Chinese,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Spanish,English": 0.21333333333333335, "Filipino,Vietnamese,Malay,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay": 0.26666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,English": 0.20666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.3333333333333333, "Filipino,Chinese,Malay,English": 0.22, "Filipino,Chinese,Indonesian,English": 0.21333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.3466666666666667, "Filipino,Spanish,Malay,English": 0.21333333333333335, "Filipino,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Spanish,English": 0.16666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.3, "Vietnamese,Chinese,Malay,English": 0.16, "Vietnamese,Chinese,Indonesian,English": 0.16, "Vietnamese,Spanish,Malay,Indonesian": 0.28, "Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.26, "Chinese,Spanish,Malay,English": 0.18666666666666668, "Chinese,Spanish,Indonesian,English": 0.18, "Chinese,Malay,Indonesian,English": 0.22666666666666666, "Spanish,Malay,Indonesian,English": 0.22 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Spanish,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, "Filipino,Vietnamese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.18, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, "Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, "AC3_2": 0.3430413027743999, "AC3_3": 0.2902864451977339, "AC3_4": 0.24322453232950383, "AC3_5": 0.2030990059445401, "AC3_6": 0.16992103369530082, "AC3_7": 0.14293478256791262 }, "prompt_2": { "overall_acc": 0.25428571428571434, "language_acc": { "Filipino": 0.2733333333333333, "Vietnamese": 0.24666666666666667, "Chinese": 0.22666666666666666, "Spanish": 0.26, "Malay": 0.23333333333333334, "Indonesian": 0.26, "English": 0.28 }, "consistency_score_2": 0.5292063492063492, "consistency_score_3": 0.3257142857142856, "consistency_score_4": 0.20933333333333332, "consistency_score_5": 0.13555555555555554, "consistency_score_6": 0.08666666666666667, "consistency_score_7": 0.05333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.58, "Filipino,Chinese": 0.4666666666666667, "Filipino,Spanish": 0.5933333333333334, "Filipino,Malay": 0.6266666666666667, "Filipino,Indonesian": 0.6133333333333333, "Filipino,English": 0.44, "Vietnamese,Chinese": 0.44, "Vietnamese,Spanish": 0.5066666666666667, "Vietnamese,Malay": 0.5866666666666667, "Vietnamese,Indonesian": 0.5933333333333334, "Vietnamese,English": 0.4666666666666667, "Chinese,Spanish": 0.5, "Chinese,Malay": 0.43333333333333335, "Chinese,Indonesian": 0.4666666666666667, "Chinese,English": 0.54, "Spanish,Malay": 0.5933333333333334, "Spanish,Indonesian": 0.6066666666666667, "Spanish,English": 0.5533333333333333, "Malay,Indonesian": 0.6066666666666667, "Malay,English": 0.41333333333333333, "Indonesian,English": 0.4866666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.28, "Filipino,Vietnamese,Spanish": 0.3466666666666667, "Filipino,Vietnamese,Malay": 0.41333333333333333, "Filipino,Vietnamese,Indonesian": 0.4, "Filipino,Vietnamese,English": 0.26666666666666666, "Filipino,Chinese,Spanish": 0.31333333333333335, "Filipino,Chinese,Malay": 0.3, "Filipino,Chinese,Indonesian": 0.3, "Filipino,Chinese,English": 0.26, "Filipino,Spanish,Malay": 0.4266666666666667, "Filipino,Spanish,Indonesian": 0.41333333333333333, "Filipino,Spanish,English": 0.32, "Filipino,Malay,Indonesian": 0.4533333333333333, "Filipino,Malay,English": 0.28, "Filipino,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish": 0.26666666666666666, "Vietnamese,Chinese,Malay": 0.26666666666666666, "Vietnamese,Chinese,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,English": 0.2733333333333333, "Vietnamese,Spanish,Malay": 0.37333333333333335, "Vietnamese,Spanish,Indonesian": 0.36666666666666664, "Vietnamese,Spanish,English": 0.3, "Vietnamese,Malay,Indonesian": 0.4266666666666667, "Vietnamese,Malay,English": 0.2733333333333333, "Vietnamese,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay": 0.3, "Chinese,Spanish,Indonesian": 0.32, "Chinese,Spanish,English": 0.34, "Chinese,Malay,Indonesian": 0.28, "Chinese,Malay,English": 0.24, "Chinese,Indonesian,English": 0.29333333333333333, "Spanish,Malay,Indonesian": 0.4266666666666667, "Spanish,Malay,English": 0.31333333333333335, "Spanish,Indonesian,English": 0.36, "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.2, "Filipino,Vietnamese,Chinese,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay": 0.2, "Filipino,Chinese,Spanish,Indonesian": 0.19333333333333333, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.20666666666666667, "Filipino,Chinese,Malay,English": 0.16, "Filipino,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.32666666666666666, "Filipino,Spanish,Malay,English": 0.21333333333333335, "Filipino,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Spanish,English": 0.18, "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Malay,English": 0.14666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian": 0.29333333333333333, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.21333333333333335, "Vietnamese,Malay,Indonesian,English": 0.22, "Chinese,Spanish,Malay,Indonesian": 0.2, "Chinese,Spanish,Malay,English": 0.19333333333333333, "Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Chinese,Malay,Indonesian,English": 0.15333333333333332, "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14, "Filipino,Vietnamese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.12, "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.12, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, "Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.08, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 } }, "AC3_2": 0.34351238708280596, "AC3_3": 0.28560168890220095, "AC3_4": 0.22963023824134926, "AC3_5": 0.17684039083411668, "AC3_6": 0.12927374297884428, "AC3_7": 0.08817337458433994 }, "prompt_3": { "overall_acc": 0.2571428571428572, "language_acc": { "Filipino": 0.23333333333333334, "Vietnamese": 0.22666666666666666, "Chinese": 0.26666666666666666, "Spanish": 0.22666666666666666, "Malay": 0.24, "Indonesian": 0.28, "English": 0.32666666666666666 }, "consistency_score_2": 0.5425396825396825, "consistency_score_3": 0.34609523809523807, "consistency_score_4": 0.2375238095238095, "consistency_score_5": 0.17111111111111116, "consistency_score_6": 0.1285714285714286, "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5666666666666667, "Filipino,Chinese": 0.4866666666666667, "Filipino,Spanish": 0.6333333333333333, "Filipino,Malay": 0.6266666666666667, "Filipino,Indonesian": 0.64, "Filipino,English": 0.47333333333333333, "Vietnamese,Chinese": 0.42, "Vietnamese,Spanish": 0.56, "Vietnamese,Malay": 0.5333333333333333, "Vietnamese,Indonesian": 0.5466666666666666, "Vietnamese,English": 0.4666666666666667, "Chinese,Spanish": 0.49333333333333335, "Chinese,Malay": 0.5133333333333333, "Chinese,Indonesian": 0.43333333333333335, "Chinese,English": 0.56, "Spanish,Malay": 0.6066666666666667, "Spanish,Indonesian": 0.6466666666666666, "Spanish,English": 0.54, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.4866666666666667, "Indonesian,English": 0.5133333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.26666666666666666, "Filipino,Vietnamese,Spanish": 0.38666666666666666, "Filipino,Vietnamese,Malay": 0.38, "Filipino,Vietnamese,Indonesian": 0.4, "Filipino,Vietnamese,English": 0.29333333333333333, "Filipino,Chinese,Spanish": 0.32, "Filipino,Chinese,Malay": 0.36, "Filipino,Chinese,Indonesian": 0.31333333333333335, "Filipino,Chinese,English": 0.3, "Filipino,Spanish,Malay": 0.44666666666666666, "Filipino,Spanish,Indonesian": 0.4866666666666667, "Filipino,Spanish,English": 0.35333333333333333, "Filipino,Malay,Indonesian": 0.47333333333333333, "Filipino,Malay,English": 0.34, "Filipino,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish": 0.26666666666666666, "Vietnamese,Chinese,Malay": 0.28, "Vietnamese,Chinese,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,English": 0.2733333333333333, "Vietnamese,Spanish,Malay": 0.36666666666666664, "Vietnamese,Spanish,Indonesian": 0.4, "Vietnamese,Spanish,English": 0.32666666666666666, "Vietnamese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Malay,English": 0.29333333333333333, "Vietnamese,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay": 0.34, "Chinese,Spanish,Indonesian": 0.32, "Chinese,Spanish,English": 0.32666666666666666, "Chinese,Malay,Indonesian": 0.34, "Chinese,Malay,English": 0.32666666666666666, "Chinese,Indonesian,English": 0.2866666666666667, "Spanish,Malay,Indonesian": 0.47333333333333333, "Spanish,Malay,English": 0.36, "Spanish,Indonesian,English": 0.37333333333333335, "Malay,Indonesian,English": 0.36 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,English": 0.18, "Filipino,Vietnamese,Spanish,Malay": 0.29333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.32, "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Malay,English": 0.22, "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay": 0.25333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.24, "Filipino,Chinese,Spanish,English": 0.20666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, "Filipino,Chinese,Malay,English": 0.22666666666666666, "Filipino,Chinese,Indonesian,English": 0.21333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.36666666666666664, "Filipino,Spanish,Malay,English": 0.2733333333333333, "Filipino,Spanish,Indonesian,English": 0.28, "Filipino,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay": 0.2, "Vietnamese,Chinese,Spanish,Indonesian": 0.2, "Vietnamese,Chinese,Spanish,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,Malay,English": 0.19333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.3, "Vietnamese,Spanish,Malay,English": 0.22666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.26, "Chinese,Spanish,Malay,English": 0.24, "Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Chinese,Malay,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian,English": 0.28 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.14, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.16, "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, "Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian,English": 0.18 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, "AC3_2": 0.34891396808431036, "AC3_3": 0.2950606702564686, "AC3_4": 0.24694427631291485, "AC3_5": 0.20548554480005346, "AC3_6": 0.17142857138412698, "AC3_7": 0.14399999995968 }, "prompt_4": { "overall_acc": 0.25047619047619046, "language_acc": { "Filipino": 0.26666666666666666, "Vietnamese": 0.26, "Chinese": 0.24666666666666667, "Spanish": 0.26666666666666666, "Malay": 0.2, "Indonesian": 0.23333333333333334, "English": 0.28 }, "consistency_score_2": 0.5377777777777778, "consistency_score_3": 0.3401904761904762, "consistency_score_4": 0.24019047619047618, "consistency_score_5": 0.18603174603174605, "consistency_score_6": 0.1542857142857143, "consistency_score_7": 0.13333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5266666666666666, "Filipino,Chinese": 0.4533333333333333, "Filipino,Spanish": 0.6066666666666667, "Filipino,Malay": 0.64, "Filipino,Indonesian": 0.6933333333333334, "Filipino,English": 0.5133333333333333, "Vietnamese,Chinese": 0.5733333333333334, "Vietnamese,Spanish": 0.5066666666666667, "Vietnamese,Malay": 0.52, "Vietnamese,Indonesian": 0.5533333333333333, "Vietnamese,English": 0.42, "Chinese,Spanish": 0.5133333333333333, "Chinese,Malay": 0.52, "Chinese,Indonesian": 0.5333333333333333, "Chinese,English": 0.47333333333333333, "Spanish,Malay": 0.5733333333333334, "Spanish,Indonesian": 0.6, "Spanish,English": 0.5266666666666666, "Malay,Indonesian": 0.66, "Malay,English": 0.44, "Indonesian,English": 0.44666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.32, "Filipino,Vietnamese,Spanish": 0.3466666666666667, "Filipino,Vietnamese,Malay": 0.35333333333333333, "Filipino,Vietnamese,Indonesian": 0.4, "Filipino,Vietnamese,English": 0.28, "Filipino,Chinese,Spanish": 0.31333333333333335, "Filipino,Chinese,Malay": 0.3333333333333333, "Filipino,Chinese,Indonesian": 0.37333333333333335, "Filipino,Chinese,English": 0.26666666666666666, "Filipino,Spanish,Malay": 0.43333333333333335, "Filipino,Spanish,Indonesian": 0.46, "Filipino,Spanish,English": 0.35333333333333333, "Filipino,Malay,Indonesian": 0.5133333333333333, "Filipino,Malay,English": 0.34, "Filipino,Indonesian,English": 0.36, "Vietnamese,Chinese,Spanish": 0.34, "Vietnamese,Chinese,Malay": 0.3466666666666667, "Vietnamese,Chinese,Indonesian": 0.36666666666666664, "Vietnamese,Chinese,English": 0.2866666666666667, "Vietnamese,Spanish,Malay": 0.3333333333333333, "Vietnamese,Spanish,Indonesian": 0.36, "Vietnamese,Spanish,English": 0.28, "Vietnamese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Malay,English": 0.24, "Vietnamese,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay": 0.3333333333333333, "Chinese,Spanish,Indonesian": 0.35333333333333333, "Chinese,Spanish,English": 0.30666666666666664, "Chinese,Malay,Indonesian": 0.3933333333333333, "Chinese,Malay,English": 0.25333333333333335, "Chinese,Indonesian,English": 0.2733333333333333, "Spanish,Malay,Indonesian": 0.44, "Spanish,Malay,English": 0.3, "Spanish,Indonesian,English": 0.31333333333333335, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.28, "Filipino,Vietnamese,Chinese,English": 0.18, "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.2, "Filipino,Chinese,Spanish,Malay": 0.25333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.26, "Filipino,Chinese,Spanish,English": 0.22666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, "Filipino,Chinese,Malay,English": 0.21333333333333335, "Filipino,Chinese,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian": 0.36, "Filipino,Spanish,Malay,English": 0.24, "Filipino,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Indonesian": 0.26, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.28, "Vietnamese,Chinese,Malay,English": 0.18, "Vietnamese,Chinese,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, "Vietnamese,Spanish,Malay,English": 0.19333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Vietnamese,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian": 0.28, "Chinese,Spanish,Malay,English": 0.21333333333333335, "Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Chinese,Malay,Indonesian,English": 0.22, "Spanish,Malay,Indonesian,English": 0.23333333333333334 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Malay,Indonesian,English": 0.2, "Filipino,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, "Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 } }, "AC3_2": 0.34176936497208055, "AC3_3": 0.2885201393839017, "AC3_4": 0.2452255249426277, "AC3_5": 0.21349679648788636, "AC3_6": 0.19095126045702548, "AC3_7": 0.17402812236987686 }, "prompt_5": { "overall_acc": 0.23333333333333336, "language_acc": { "Filipino": 0.22666666666666666, "Vietnamese": 0.28, "Chinese": 0.18, "Spanish": 0.20666666666666667, "Malay": 0.26666666666666666, "Indonesian": 0.23333333333333334, "English": 0.24 }, "consistency_score_2": 0.5333333333333333, "consistency_score_3": 0.338095238095238, "consistency_score_4": 0.23161904761904764, "consistency_score_5": 0.16730158730158734, "consistency_score_6": 0.12476190476190477, "consistency_score_7": 0.09333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4666666666666667, "Filipino,Chinese": 0.5266666666666666, "Filipino,Spanish": 0.62, "Filipino,Malay": 0.5866666666666667, "Filipino,Indonesian": 0.6266666666666667, "Filipino,English": 0.5066666666666667, "Vietnamese,Chinese": 0.5466666666666666, "Vietnamese,Spanish": 0.52, "Vietnamese,Malay": 0.5333333333333333, "Vietnamese,Indonesian": 0.5666666666666667, "Vietnamese,English": 0.56, "Chinese,Spanish": 0.5066666666666667, "Chinese,Malay": 0.48, "Chinese,Indonesian": 0.5266666666666666, "Chinese,English": 0.5333333333333333, "Spanish,Malay": 0.5066666666666667, "Spanish,Indonesian": 0.5333333333333333, "Spanish,English": 0.52, "Malay,Indonesian": 0.62, "Malay,English": 0.43333333333333335, "Indonesian,English": 0.48 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.31333333333333335, "Filipino,Vietnamese,Spanish": 0.3333333333333333, "Filipino,Vietnamese,Malay": 0.32666666666666666, "Filipino,Vietnamese,Indonesian": 0.36, "Filipino,Vietnamese,English": 0.30666666666666664, "Filipino,Chinese,Spanish": 0.3466666666666667, "Filipino,Chinese,Malay": 0.32666666666666666, "Filipino,Chinese,Indonesian": 0.38, "Filipino,Chinese,English": 0.32, "Filipino,Spanish,Malay": 0.37333333333333335, "Filipino,Spanish,Indonesian": 0.41333333333333333, "Filipino,Spanish,English": 0.3466666666666667, "Filipino,Malay,Indonesian": 0.44, "Filipino,Malay,English": 0.30666666666666664, "Filipino,Indonesian,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish": 0.3333333333333333, "Vietnamese,Chinese,Malay": 0.31333333333333335, "Vietnamese,Chinese,Indonesian": 0.37333333333333335, "Vietnamese,Chinese,English": 0.38, "Vietnamese,Spanish,Malay": 0.32, "Vietnamese,Spanish,Indonesian": 0.35333333333333333, "Vietnamese,Spanish,English": 0.34, "Vietnamese,Malay,Indonesian": 0.4, "Vietnamese,Malay,English": 0.32, "Vietnamese,Indonesian,English": 0.3466666666666667, "Chinese,Spanish,Malay": 0.26666666666666666, "Chinese,Spanish,Indonesian": 0.32666666666666666, "Chinese,Spanish,English": 0.31333333333333335, "Chinese,Malay,Indonesian": 0.36, "Chinese,Malay,English": 0.2733333333333333, "Chinese,Indonesian,English": 0.31333333333333335, "Spanish,Malay,Indonesian": 0.36, "Spanish,Malay,English": 0.26, "Spanish,Indonesian,English": 0.32, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Indonesian,English": 0.25333333333333335, "Filipino,Chinese,Spanish,Malay": 0.2, "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Chinese,Spanish,English": 0.21333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, "Filipino,Chinese,Malay,English": 0.20666666666666667, "Filipino,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.2866666666666667, "Filipino,Spanish,Malay,English": 0.20666666666666667, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.2, "Vietnamese,Chinese,Spanish,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,Spanish,English": 0.23333333333333334, "Vietnamese,Chinese,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Malay,English": 0.21333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.24666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.26, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Vietnamese,Malay,Indonesian,English": 0.25333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, "Chinese,Spanish,Malay,English": 0.16666666666666666, "Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Chinese,Malay,Indonesian,English": 0.20666666666666667, "Spanish,Malay,Indonesian,English": 0.22 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, "Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 } }, "AC3_2": 0.32463768111707625, "AC3_3": 0.2761111110627917, "AC3_4": 0.23247303012888915, "AC3_5": 0.19487585837713747, "AC3_6": 0.16258865243686577, "AC3_7": 0.133333333292517 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.26461038961038963, "language_acc": { "Indonesian": 0.26136363636363635, "English": 0.26136363636363635, "Filipino": 0.26704545454545453, "Spanish": 0.2897727272727273, "Chinese": 0.22727272727272727, "Malay": 0.2897727272727273, "Vietnamese": 0.2556818181818182 }, "consistency_score_2": 0.38609307359307365, "consistency_score_3": 0.18944805194805198, "consistency_score_4": 0.10487012987012986, "consistency_score_5": 0.06114718614718615, "consistency_score_6": 0.036525974025974024, "consistency_score_7": 0.022727272727272728, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.375, "Indonesian,Filipino": 0.48295454545454547, "Indonesian,Spanish": 0.4772727272727273, "Indonesian,Chinese": 0.2215909090909091, "Indonesian,Malay": 0.5227272727272727, "Indonesian,Vietnamese": 0.5284090909090909, "English,Filipino": 0.36363636363636365, "English,Spanish": 0.42613636363636365, "English,Chinese": 0.3465909090909091, "English,Malay": 0.3068181818181818, "English,Vietnamese": 0.3465909090909091, "Filipino,Spanish": 0.4943181818181818, "Filipino,Chinese": 0.2159090909090909, "Filipino,Malay": 0.44886363636363635, "Filipino,Vietnamese": 0.5227272727272727, "Spanish,Chinese": 0.25, "Spanish,Malay": 0.4318181818181818, "Spanish,Vietnamese": 0.44886363636363635, "Chinese,Malay": 0.19318181818181818, "Chinese,Vietnamese": 0.26136363636363635, "Malay,Vietnamese": 0.4431818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.2215909090909091, "Indonesian,English,Spanish": 0.23863636363636365, "Indonesian,English,Chinese": 0.10227272727272728, "Indonesian,English,Malay": 0.20454545454545456, "Indonesian,English,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish": 0.2897727272727273, "Indonesian,Filipino,Chinese": 0.14772727272727273, "Indonesian,Filipino,Malay": 0.26704545454545453, "Indonesian,Filipino,Vietnamese": 0.32386363636363635, "Indonesian,Spanish,Chinese": 0.125, "Indonesian,Spanish,Malay": 0.26704545454545453, "Indonesian,Spanish,Vietnamese": 0.29545454545454547, "Indonesian,Chinese,Malay": 0.13068181818181818, "Indonesian,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Malay,Vietnamese": 0.3068181818181818, "English,Filipino,Spanish": 0.2215909090909091, "English,Filipino,Chinese": 0.10795454545454546, "English,Filipino,Malay": 0.1875, "English,Filipino,Vietnamese": 0.2215909090909091, "English,Spanish,Chinese": 0.13068181818181818, "English,Spanish,Malay": 0.1875, "English,Spanish,Vietnamese": 0.19318181818181818, "English,Chinese,Malay": 0.08522727272727272, "English,Chinese,Vietnamese": 0.10227272727272728, "English,Malay,Vietnamese": 0.1875, "Filipino,Spanish,Chinese": 0.11363636363636363, "Filipino,Spanish,Malay": 0.26136363636363635, "Filipino,Spanish,Vietnamese": 0.3068181818181818, "Filipino,Chinese,Malay": 0.11931818181818182, "Filipino,Chinese,Vietnamese": 0.13068181818181818, "Filipino,Malay,Vietnamese": 0.2784090909090909, "Spanish,Chinese,Malay": 0.09090909090909091, "Spanish,Chinese,Vietnamese": 0.10227272727272728, "Spanish,Malay,Vietnamese": 0.23295454545454544, "Chinese,Malay,Vietnamese": 0.10795454545454546 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.14204545454545456, "Indonesian,English,Filipino,Chinese": 0.06818181818181818, "Indonesian,English,Filipino,Malay": 0.125, "Indonesian,English,Filipino,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese": 0.06818181818181818, "Indonesian,English,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,English,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Chinese": 0.09659090909090909, "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, "Indonesian,Filipino,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese,Malay": 0.07386363636363637, "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese": 0.0625, "English,Filipino,Spanish,Malay": 0.13068181818181818, "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.06818181818181818, "English,Filipino,Malay,Vietnamese": 0.13068181818181818, "English,Spanish,Chinese,Malay": 0.05113636363636364, "English,Spanish,Chinese,Vietnamese": 0.045454545454545456, "English,Spanish,Malay,Vietnamese": 0.11363636363636363, "English,Chinese,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese,Malay": 0.0625, "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, "English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 } }, "AC3_2": 0.3140116639634068, "AC3_3": 0.22080824069168092, "AC3_4": 0.15020941269072685, "AC3_5": 0.09933878410895389, "AC3_6": 0.06419119961462726, "AC3_7": 0.041859270658262025 }, "prompt_2": { "overall_acc": 0.26623376623376627, "language_acc": { "Indonesian": 0.25, "English": 0.2897727272727273, "Filipino": 0.24431818181818182, "Spanish": 0.2784090909090909, "Chinese": 0.25, "Malay": 0.26704545454545453, "Vietnamese": 0.2840909090909091 }, "consistency_score_2": 0.3663419913419913, "consistency_score_3": 0.15616883116883118, "consistency_score_4": 0.07175324675324674, "consistency_score_5": 0.03219696969696971, "consistency_score_6": 0.011363636363636364, "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3465909090909091, "Indonesian,Filipino": 0.5340909090909091, "Indonesian,Spanish": 0.3977272727272727, "Indonesian,Chinese": 0.1590909090909091, "Indonesian,Malay": 0.5056818181818182, "Indonesian,Vietnamese": 0.42045454545454547, "English,Filipino": 0.24431818181818182, "English,Spanish": 0.4715909090909091, "English,Chinese": 0.4943181818181818, "English,Malay": 0.29545454545454547, "English,Vietnamese": 0.36363636363636365, "Filipino,Spanish": 0.35795454545454547, "Filipino,Chinese": 0.08522727272727272, "Filipino,Malay": 0.5, "Filipino,Vietnamese": 0.4715909090909091, "Spanish,Chinese": 0.42045454545454547, "Spanish,Malay": 0.3977272727272727, "Spanish,Vietnamese": 0.42613636363636365, "Chinese,Malay": 0.14772727272727273, "Chinese,Vietnamese": 0.22727272727272727, "Malay,Vietnamese": 0.42613636363636365 }, "3_combine": { "Indonesian,English,Filipino": 0.1534090909090909, "Indonesian,English,Spanish": 0.20454545454545456, "Indonesian,English,Chinese": 0.09659090909090909, "Indonesian,English,Malay": 0.18181818181818182, "Indonesian,English,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish": 0.23295454545454544, "Indonesian,Filipino,Chinese": 0.03977272727272727, "Indonesian,Filipino,Malay": 0.3125, "Indonesian,Filipino,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Chinese": 0.10227272727272728, "Indonesian,Spanish,Malay": 0.23863636363636365, "Indonesian,Spanish,Vietnamese": 0.22727272727272727, "Indonesian,Chinese,Malay": 0.07386363636363637, "Indonesian,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,Malay,Vietnamese": 0.23295454545454544, "English,Filipino,Spanish": 0.13068181818181818, "English,Filipino,Chinese": 0.028409090909090908, "English,Filipino,Malay": 0.13636363636363635, "English,Filipino,Vietnamese": 0.1534090909090909, "English,Spanish,Chinese": 0.2556818181818182, "English,Spanish,Malay": 0.1875, "English,Spanish,Vietnamese": 0.19318181818181818, "English,Chinese,Malay": 0.09090909090909091, "English,Chinese,Vietnamese": 0.13636363636363635, "English,Malay,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Chinese": 0.03977272727272727, "Filipino,Spanish,Malay": 0.23295454545454544, "Filipino,Spanish,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay": 0.022727272727272728, "Filipino,Chinese,Vietnamese": 0.0625, "Filipino,Malay,Vietnamese": 0.2897727272727273, "Spanish,Chinese,Malay": 0.10227272727272728, "Spanish,Chinese,Vietnamese": 0.125, "Spanish,Malay,Vietnamese": 0.2159090909090909, "Chinese,Malay,Vietnamese": 0.05113636363636364 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.09659090909090909, "Indonesian,English,Filipino,Chinese": 0.005681818181818182, "Indonesian,English,Filipino,Malay": 0.09659090909090909, "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese": 0.0625, "Indonesian,English,Spanish,Malay": 0.125, "Indonesian,English,Spanish,Vietnamese": 0.10795454545454546, "Indonesian,English,Chinese,Malay": 0.05113636363636364, "Indonesian,English,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,English,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Malay": 0.1590909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Chinese,Malay": 0.011363636363636364, "Indonesian,Filipino,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Malay,Vietnamese": 0.1875, "Indonesian,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese": 0.017045454545454544, "English,Filipino,Spanish,Malay": 0.08522727272727272, "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay": 0.005681818181818182, "English,Filipino,Chinese,Vietnamese": 0.022727272727272728, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.07386363636363637, "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, "English,Spanish,Malay,Vietnamese": 0.09090909090909091, "English,Chinese,Malay,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.0625, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.045454545454545456, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, "AC3_2": 0.3083665692245794, "AC3_3": 0.19686155500096064, "AC3_4": 0.11304065767580032, "AC3_5": 0.05744663313001279, "AC3_6": 0.02179691652590665, "AC3_7": 0.0 }, "prompt_3": { "overall_acc": 0.26461038961038963, "language_acc": { "Indonesian": 0.26136363636363635, "English": 0.2784090909090909, "Filipino": 0.24431818181818182, "Spanish": 0.2840909090909091, "Chinese": 0.2840909090909091, "Malay": 0.25, "Vietnamese": 0.25 }, "consistency_score_2": 0.40854978354978355, "consistency_score_3": 0.21217532467532468, "consistency_score_4": 0.12045454545454548, "consistency_score_5": 0.06737012987012987, "consistency_score_6": 0.03327922077922078, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.36363636363636365, "Indonesian,Filipino": 0.6022727272727273, "Indonesian,Spanish": 0.5227272727272727, "Indonesian,Chinese": 0.26704545454545453, "Indonesian,Malay": 0.5568181818181818, "Indonesian,Vietnamese": 0.48863636363636365, "English,Filipino": 0.29545454545454547, "English,Spanish": 0.45454545454545453, "English,Chinese": 0.4090909090909091, "English,Malay": 0.29545454545454547, "English,Vietnamese": 0.375, "Filipino,Spanish": 0.4715909090909091, "Filipino,Chinese": 0.25, "Filipino,Malay": 0.5113636363636364, "Filipino,Vietnamese": 0.48863636363636365, "Spanish,Chinese": 0.3465909090909091, "Spanish,Malay": 0.44886363636363635, "Spanish,Vietnamese": 0.5, "Chinese,Malay": 0.23863636363636365, "Chinese,Vietnamese": 0.2727272727272727, "Malay,Vietnamese": 0.42045454545454547 }, "3_combine": { "Indonesian,English,Filipino": 0.2159090909090909, "Indonesian,English,Spanish": 0.2727272727272727, "Indonesian,English,Chinese": 0.125, "Indonesian,English,Malay": 0.23863636363636365, "Indonesian,English,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish": 0.35795454545454547, "Indonesian,Filipino,Chinese": 0.17045454545454544, "Indonesian,Filipino,Malay": 0.38636363636363635, "Indonesian,Filipino,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Chinese": 0.17045454545454544, "Indonesian,Spanish,Malay": 0.3352272727272727, "Indonesian,Spanish,Vietnamese": 0.3352272727272727, "Indonesian,Chinese,Malay": 0.1590909090909091, "Indonesian,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Malay,Vietnamese": 0.2840909090909091, "English,Filipino,Spanish": 0.20454545454545456, "English,Filipino,Chinese": 0.09090909090909091, "English,Filipino,Malay": 0.17613636363636365, "English,Filipino,Vietnamese": 0.19318181818181818, "English,Spanish,Chinese": 0.18181818181818182, "English,Spanish,Malay": 0.2159090909090909, "English,Spanish,Vietnamese": 0.2556818181818182, "English,Chinese,Malay": 0.09659090909090909, "English,Chinese,Vietnamese": 0.13068181818181818, "English,Malay,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Chinese": 0.1534090909090909, "Filipino,Spanish,Malay": 0.2784090909090909, "Filipino,Spanish,Vietnamese": 0.2897727272727273, "Filipino,Chinese,Malay": 0.1590909090909091, "Filipino,Chinese,Vietnamese": 0.14204545454545456, "Filipino,Malay,Vietnamese": 0.26136363636363635, "Spanish,Chinese,Malay": 0.14772727272727273, "Spanish,Chinese,Vietnamese": 0.1534090909090909, "Spanish,Malay,Vietnamese": 0.26136363636363635, "Chinese,Malay,Vietnamese": 0.11931818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1590909090909091, "Indonesian,English,Filipino,Chinese": 0.0625, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.13636363636363635, "Indonesian,English,Spanish,Chinese": 0.09659090909090909, "Indonesian,English,Spanish,Malay": 0.17613636363636365, "Indonesian,English,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,English,Chinese,Malay": 0.07954545454545454, "Indonesian,English,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,English,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Malay": 0.23863636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Chinese,Malay": 0.125, "Indonesian,Filipino,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Chinese": 0.06818181818181818, "English,Filipino,Spanish,Malay": 0.13636363636363635, "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, "English,Filipino,Chinese,Malay": 0.05113636363636364, "English,Filipino,Chinese,Vietnamese": 0.0625, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.06818181818181818, "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, "English,Spanish,Malay,Vietnamese": 0.13636363636363635, "English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, "AC3_2": 0.32119106772695516, "AC3_3": 0.23550955338661372, "AC3_4": 0.16554882718374375, "AC3_5": 0.10739688181440923, "AC3_6": 0.05912275733179299, "AC3_7": 0.021791443842371187 }, "prompt_4": { "overall_acc": 0.26461038961038963, "language_acc": { "Indonesian": 0.25, "English": 0.2897727272727273, "Filipino": 0.23863636363636365, "Spanish": 0.29545454545454547, "Chinese": 0.26704545454545453, "Malay": 0.26136363636363635, "Vietnamese": 0.25 }, "consistency_score_2": 0.4323593073593074, "consistency_score_3": 0.2308441558441558, "consistency_score_4": 0.13636363636363638, "consistency_score_5": 0.08306277056277056, "consistency_score_6": 0.04951298701298701, "consistency_score_7": 0.028409090909090908, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.35795454545454547, "Indonesian,Filipino": 0.6136363636363636, "Indonesian,Spanish": 0.5568181818181818, "Indonesian,Chinese": 0.21022727272727273, "Indonesian,Malay": 0.5795454545454546, "Indonesian,Vietnamese": 0.5340909090909091, "English,Filipino": 0.32386363636363635, "English,Spanish": 0.48863636363636365, "English,Chinese": 0.4772727272727273, "English,Malay": 0.3125, "English,Vietnamese": 0.39204545454545453, "Filipino,Spanish": 0.5625, "Filipino,Chinese": 0.20454545454545456, "Filipino,Malay": 0.5965909090909091, "Filipino,Vietnamese": 0.5454545454545454, "Spanish,Chinese": 0.36363636363636365, "Spanish,Malay": 0.48295454545454547, "Spanish,Vietnamese": 0.5227272727272727, "Chinese,Malay": 0.17613636363636365, "Chinese,Vietnamese": 0.2556818181818182, "Malay,Vietnamese": 0.5227272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.2556818181818182, "Indonesian,English,Spanish": 0.2784090909090909, "Indonesian,English,Chinese": 0.10795454545454546, "Indonesian,English,Malay": 0.22727272727272727, "Indonesian,English,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Spanish": 0.4147727272727273, "Indonesian,Filipino,Chinese": 0.1590909090909091, "Indonesian,Filipino,Malay": 0.4090909090909091, "Indonesian,Filipino,Vietnamese": 0.3806818181818182, "Indonesian,Spanish,Chinese": 0.13068181818181818, "Indonesian,Spanish,Malay": 0.36363636363636365, "Indonesian,Spanish,Vietnamese": 0.35795454545454547, "Indonesian,Chinese,Malay": 0.125, "Indonesian,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish": 0.25, "English,Filipino,Chinese": 0.09659090909090909, "English,Filipino,Malay": 0.2159090909090909, "English,Filipino,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese": 0.21022727272727273, "English,Spanish,Malay": 0.23295454545454544, "English,Spanish,Vietnamese": 0.26704545454545453, "English,Chinese,Malay": 0.08522727272727272, "English,Chinese,Vietnamese": 0.14772727272727273, "English,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese": 0.1534090909090909, "Filipino,Spanish,Malay": 0.375, "Filipino,Spanish,Vietnamese": 0.36363636363636365, "Filipino,Chinese,Malay": 0.13068181818181818, "Filipino,Chinese,Vietnamese": 0.13068181818181818, "Filipino,Malay,Vietnamese": 0.36363636363636365, "Spanish,Chinese,Malay": 0.11931818181818182, "Spanish,Chinese,Vietnamese": 0.14204545454545456, "Spanish,Malay,Vietnamese": 0.3181818181818182, "Chinese,Malay,Vietnamese": 0.10795454545454546 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.20454545454545456, "Indonesian,English,Filipino,Chinese": 0.07954545454545454, "Indonesian,English,Filipino,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese": 0.06818181818181818, "Indonesian,English,Spanish,Malay": 0.1875, "Indonesian,English,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,English,Chinese,Malay": 0.0625, "Indonesian,English,Chinese,Vietnamese": 0.0625, "Indonesian,English,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Malay": 0.29545454545454547, "Indonesian,Filipino,Spanish,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Spanish,Malay,Vietnamese": 0.25, "Indonesian,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Chinese": 0.07386363636363637, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.0625, "English,Filipino,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay": 0.0625, "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, "English,Spanish,Malay,Vietnamese": 0.18181818181818182, "English,Chinese,Malay,Vietnamese": 0.0625, "Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.26136363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 } }, "AC3_2": 0.32829767277696675, "AC3_3": 0.24657665396463196, "AC3_4": 0.17997791677513691, "AC3_5": 0.12643640402649675, "AC3_6": 0.08341722873951041, "AC3_7": 0.05130949381275885 }, "prompt_5": { "overall_acc": 0.2621753246753247, "language_acc": { "Indonesian": 0.2215909090909091, "English": 0.2727272727272727, "Filipino": 0.2727272727272727, "Spanish": 0.29545454545454547, "Chinese": 0.2727272727272727, "Malay": 0.25, "Vietnamese": 0.25 }, "consistency_score_2": 0.4388528138528139, "consistency_score_3": 0.24464285714285705, "consistency_score_4": 0.1547077922077922, "consistency_score_5": 0.10551948051948054, "consistency_score_6": 0.07386363636363637, "consistency_score_7": 0.05113636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3409090909090909, "Indonesian,Filipino": 0.6022727272727273, "Indonesian,Spanish": 0.5397727272727273, "Indonesian,Chinese": 0.26704545454545453, "Indonesian,Malay": 0.5965909090909091, "Indonesian,Vietnamese": 0.5284090909090909, "English,Filipino": 0.3693181818181818, "English,Spanish": 0.48295454545454547, "English,Chinese": 0.4715909090909091, "English,Malay": 0.30113636363636365, "English,Vietnamese": 0.42045454545454547, "Filipino,Spanish": 0.5681818181818182, "Filipino,Chinese": 0.32386363636363635, "Filipino,Malay": 0.5113636363636364, "Filipino,Vietnamese": 0.5113636363636364, "Spanish,Chinese": 0.35795454545454547, "Spanish,Malay": 0.4034090909090909, "Spanish,Vietnamese": 0.5738636363636364, "Chinese,Malay": 0.23863636363636365, "Chinese,Vietnamese": 0.3181818181818182, "Malay,Vietnamese": 0.48863636363636365 }, "3_combine": { "Indonesian,English,Filipino": 0.23295454545454544, "Indonesian,English,Spanish": 0.26704545454545453, "Indonesian,English,Chinese": 0.1590909090909091, "Indonesian,English,Malay": 0.21022727272727273, "Indonesian,English,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Spanish": 0.38636363636363635, "Indonesian,Filipino,Chinese": 0.2159090909090909, "Indonesian,Filipino,Malay": 0.375, "Indonesian,Filipino,Vietnamese": 0.3522727272727273, "Indonesian,Spanish,Chinese": 0.1875, "Indonesian,Spanish,Malay": 0.32386363636363635, "Indonesian,Spanish,Vietnamese": 0.35795454545454547, "Indonesian,Chinese,Malay": 0.16477272727272727, "Indonesian,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,Malay,Vietnamese": 0.36363636363636365, "English,Filipino,Spanish": 0.2784090909090909, "English,Filipino,Chinese": 0.17613636363636365, "English,Filipino,Malay": 0.20454545454545456, "English,Filipino,Vietnamese": 0.25, "English,Spanish,Chinese": 0.2159090909090909, "English,Spanish,Malay": 0.2159090909090909, "English,Spanish,Vietnamese": 0.3181818181818182, "English,Chinese,Malay": 0.14204545454545456, "English,Chinese,Vietnamese": 0.20454545454545456, "English,Malay,Vietnamese": 0.20454545454545456, "Filipino,Spanish,Chinese": 0.21022727272727273, "Filipino,Spanish,Malay": 0.2897727272727273, "Filipino,Spanish,Vietnamese": 0.3693181818181818, "Filipino,Chinese,Malay": 0.18181818181818182, "Filipino,Chinese,Vietnamese": 0.19318181818181818, "Filipino,Malay,Vietnamese": 0.30113636363636365, "Spanish,Chinese,Malay": 0.14204545454545456, "Spanish,Chinese,Vietnamese": 0.2215909090909091, "Spanish,Malay,Vietnamese": 0.2840909090909091, "Chinese,Malay,Vietnamese": 0.14772727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.19318181818181818, "Indonesian,English,Filipino,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese": 0.11931818181818182, "Indonesian,English,Spanish,Malay": 0.17045454545454544, "Indonesian,English,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,English,Chinese,Malay": 0.10227272727272728, "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Chinese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Malay": 0.23863636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.25, "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Spanish,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese": 0.125, "English,Filipino,Spanish,Malay": 0.1590909090909091, "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "English,Filipino,Chinese,Malay": 0.10795454545454546, "English,Filipino,Chinese,Vietnamese": 0.125, "English,Filipino,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay": 0.09090909090909091, "English,Spanish,Chinese,Vietnamese": 0.14204545454545456, "English,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Chinese,Malay,Vietnamese": 0.09659090909090909, "Filipino,Spanish,Chinese,Malay": 0.11931818181818182, "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.125, "Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Chinese,Malay,Vietnamese": 0.07954545454545454, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0625, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 } }, "AC3_2": 0.3282503871576349, "AC3_3": 0.25310583869811504, "AC3_4": 0.1945896295604757, "AC3_5": 0.15047590374623793, "AC3_6": 0.11525581902586854, "AC3_7": 0.08558054636929374 } }, "sg_eval": { "prompt_1": { "accuracy": 0.3300970873786408 }, "prompt_2": { "accuracy": 0.2912621359223301 }, "prompt_3": { "accuracy": 0.2815533980582524 }, "prompt_4": { "accuracy": 0.22330097087378642 }, "prompt_5": { "accuracy": 0.2621359223300971 } }, "cn_eval": { "prompt_1": { "accuracy": 0.22857142857142856 }, "prompt_2": { "accuracy": 0.2761904761904762 }, "prompt_3": { "accuracy": 0.24761904761904763 }, "prompt_4": { "accuracy": 0.2 }, "prompt_5": { "accuracy": 0.20952380952380953 } }, "us_eval": { "prompt_1": { "accuracy": 0.2897196261682243 }, "prompt_2": { "accuracy": 0.27102803738317754 }, "prompt_3": { "accuracy": 0.2803738317757009 }, "prompt_4": { "accuracy": 0.2616822429906542 }, "prompt_5": { "accuracy": 0.2336448598130841 } }, "ph_eval": { "prompt_1": { "accuracy": 0.25, "category_acc": { "brand": 0.4, "demographics": 0.4, "biology": 0.3, "history": 0.2, "literature": 0.1, "politics": 0.1, "culture": 0.2, "film": 0.4, "law": 0.3, "geography": 0.2 } }, "prompt_2": { "accuracy": 0.18, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.2, "history": 0.13333333333333333, "literature": 0.4, "politics": 0.0, "culture": 0.1, "film": 0.2, "law": 0.2, "geography": 0.2 } }, "prompt_3": { "accuracy": 0.22, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.2, "literature": 0.3, "politics": 0.0, "culture": 0.1, "film": 0.2, "law": 0.2, "geography": 0.3 } }, "prompt_4": { "accuracy": 0.24, "category_acc": { "brand": 0.1, "demographics": 0.4, "biology": 0.3, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.2, "culture": 0.3, "film": 0.1, "law": 0.2, "geography": 0.3 } }, "prompt_5": { "accuracy": 0.23, "category_acc": { "brand": 0.2, "demographics": 0.4, "biology": 0.2, "history": 0.13333333333333333, "literature": 0.3, "politics": 0.0, "culture": 0.3, "film": 0.4, "law": 0.3, "geography": 0.2 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.17364911315618559 }, "prompt_2": { "bleu_score": 0.1816094741862085 }, "prompt_3": { "bleu_score": 0.15745870540541765 }, "prompt_4": { "bleu_score": 0.1604700546938792 }, "prompt_5": { "bleu_score": 0.14500235127771283 } }, "indommlu": { "prompt_1": { "accuracy": 0.2488817678082649, "category_acc": { "History": 0.20080321285140562, "Geography": 0.24285714285714285, "Lampungic": 0.24489795918367346, "Social science": 0.3005008347245409, "Balinese": 0.2611464968152866, "Makassarese": 0.25806451612903225, "Banjarese": 0.3680555555555556, "Chemistry": 0.2072992700729927, "Biology": 0.22130177514792898, "Science": 0.24974200206398348, "Christian religion": 0.27860696517412936, "Art": 0.2628951747088186, "Islam religion": 0.22617354196301565, "Hindu religion": 0.23333333333333334, "Madurese": 0.27796610169491526, "Sport": 0.24324324324324326, "Indonesian language": 0.26774595267745954, "Physics": 0.2202020202020202, "Minangkabau culture": 0.24623115577889448, "Dayak language": 0.23853211009174313, "Sociology": 0.19556451612903225, "Economy": 0.19262295081967212, "Sundanese": 0.26534140017286084, "Javanese": 0.26310483870967744, "Civic education": 0.24177396280400573 } }, "prompt_2": { "accuracy": 0.2415381534147807, "category_acc": { "History": 0.1827309236947791, "Geography": 0.23061224489795917, "Lampungic": 0.2653061224489796, "Social science": 0.3005008347245409, "Balinese": 0.2632696390658174, "Makassarese": 0.25806451612903225, "Banjarese": 0.25, "Chemistry": 0.22189781021897811, "Biology": 0.20828402366863905, "Science": 0.24251805985552116, "Christian religion": 0.24378109452736318, "Art": 0.24459234608985025, "Islam religion": 0.22901849217638692, "Hindu religion": 0.24666666666666667, "Madurese": 0.2440677966101695, "Sport": 0.2972972972972973, "Indonesian language": 0.26058530510585304, "Physics": 0.19595959595959597, "Minangkabau culture": 0.24623115577889448, "Dayak language": 0.22018348623853212, "Sociology": 0.19153225806451613, "Economy": 0.22336065573770492, "Sundanese": 0.2808988764044944, "Javanese": 0.22983870967741934, "Civic education": 0.2145922746781116 } }, "prompt_3": { "accuracy": 0.24667868349021965, "category_acc": { "History": 0.23493975903614459, "Geography": 0.19795918367346937, "Lampungic": 0.2108843537414966, "Social science": 0.327212020033389, "Balinese": 0.24416135881104034, "Makassarese": 0.25806451612903225, "Banjarese": 0.2777777777777778, "Chemistry": 0.21897810218978103, "Biology": 0.21420118343195266, "Science": 0.25696594427244585, "Christian religion": 0.22388059701492538, "Art": 0.2778702163061564, "Islam religion": 0.25889046941678523, "Hindu religion": 0.25333333333333335, "Madurese": 0.26440677966101694, "Sport": 0.2635135135135135, "Indonesian language": 0.2562266500622665, "Physics": 0.21212121212121213, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.28440366972477066, "Sociology": 0.22782258064516128, "Economy": 0.19467213114754098, "Sundanese": 0.2757130509939499, "Javanese": 0.24193548387096775, "Civic education": 0.21316165951359084 } }, "prompt_4": { "accuracy": 0.25435609853795316, "category_acc": { "History": 0.2469879518072289, "Geography": 0.23877551020408164, "Lampungic": 0.2653061224489796, "Social science": 0.29549248747913187, "Balinese": 0.27388535031847133, "Makassarese": 0.24731182795698925, "Banjarese": 0.2916666666666667, "Chemistry": 0.24379562043795622, "Biology": 0.2106508875739645, "Science": 0.2579979360165119, "Christian religion": 0.21890547263681592, "Art": 0.27121464226289516, "Islam religion": 0.2802275960170697, "Hindu religion": 0.24, "Madurese": 0.24067796610169492, "Sport": 0.24324324324324326, "Indonesian language": 0.2714819427148194, "Physics": 0.21616161616161617, "Minangkabau culture": 0.2964824120603015, "Dayak language": 0.24770642201834864, "Sociology": 0.22983870967741934, "Economy": 0.22131147540983606, "Sundanese": 0.2783059636992221, "Javanese": 0.22983870967741934, "Civic education": 0.22603719599427755 } }, "prompt_5": { "accuracy": 0.2538887776220041, "category_acc": { "History": 0.21887550200803213, "Geography": 0.21836734693877552, "Lampungic": 0.25170068027210885, "Social science": 0.3005008347245409, "Balinese": 0.2653927813163482, "Makassarese": 0.2903225806451613, "Banjarese": 0.24305555555555555, "Chemistry": 0.22189781021897811, "Biology": 0.22603550295857988, "Science": 0.27657378740970073, "Christian religion": 0.2835820895522388, "Art": 0.27287853577371046, "Islam religion": 0.2233285917496444, "Hindu religion": 0.22, "Madurese": 0.2677966101694915, "Sport": 0.22297297297297297, "Indonesian language": 0.27179327521793273, "Physics": 0.19797979797979798, "Minangkabau culture": 0.24120603015075376, "Dayak language": 0.2018348623853211, "Sociology": 0.21370967741935484, "Economy": 0.2540983606557377, "Sundanese": 0.30682800345721695, "Javanese": 0.23286290322580644, "Civic education": 0.23605150214592274 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.22847830117086618 }, "prompt_2": { "bleu_score": 0.2250685143300185 }, "prompt_3": { "bleu_score": 0.22062239532038108 }, "prompt_4": { "bleu_score": 0.22033637777813436 }, "prompt_5": { "bleu_score": 0.21347066048139857 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.18936731971081208 }, "prompt_2": { "bleu_score": 0.18190614295595098 }, "prompt_3": { "bleu_score": 0.16718291578910777 }, "prompt_4": { "bleu_score": 0.1853209684094906 }, "prompt_5": { "bleu_score": 0.18652646523521288 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.1249721609421984 }, "prompt_2": { "bleu_score": 0.11767975443160957 }, "prompt_3": { "bleu_score": 0.12164300828346879 }, "prompt_4": { "bleu_score": 0.11784310604903017 }, "prompt_5": { "bleu_score": 0.11834728651092338 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.21558251564062889 }, "prompt_2": { "bleu_score": 0.21464399589286035 }, "prompt_3": { "bleu_score": 0.21852482863861056 }, "prompt_4": { "bleu_score": 0.2121958104676107 }, "prompt_5": { "bleu_score": 0.21493092223269536 } }, "mmlu": { "prompt_1": { "accuracy": 0.28004667444574094 }, "prompt_2": { "accuracy": 0.26837806301050177 }, "prompt_3": { "accuracy": 0.27887981330221706 }, "prompt_4": { "accuracy": 0.2660443407234539 }, "prompt_5": { "accuracy": 0.25204200700116686 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.26478369681801933, "category_acc": { "high_school_european_history": 0.34146341463414637, "business_ethics": 0.24242424242424243, "clinical_knowledge": 0.2727272727272727, "medical_genetics": 0.24242424242424243, "high_school_us_history": 0.2413793103448276, "high_school_physics": 0.23333333333333334, "high_school_world_history": 0.2711864406779661, "virology": 0.23030303030303031, "high_school_microeconomics": 0.24050632911392406, "econometrics": 0.23893805309734514, "college_computer_science": 0.20202020202020202, "high_school_biology": 0.28802588996763756, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.28825622775800713, "philosophy": 0.2645161290322581, "professional_medicine": 0.23985239852398524, "nutrition": 0.26885245901639343, "global_facts": 0.2828282828282828, "machine_learning": 0.25225225225225223, "security_studies": 0.2336065573770492, "public_relations": 0.28440366972477066, "professional_psychology": 0.25040916530278234, "prehistory": 0.25696594427244585, "anatomy": 0.35074626865671643, "human_sexuality": 0.24615384615384617, "college_medicine": 0.2616279069767442, "high_school_government_and_politics": 0.265625, "college_chemistry": 0.2222222222222222, "logical_fallacies": 0.2962962962962963, "high_school_geography": 0.3299492385786802, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.18018018018018017, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.26838235294117646, "formal_logic": 0.184, "high_school_statistics": 0.26976744186046514, "international_law": 0.25833333333333336, "high_school_mathematics": 0.26394052044609667, "high_school_computer_science": 0.24242424242424243, "conceptual_physics": 0.24786324786324787, "miscellaneous": 0.29539641943734013, "high_school_chemistry": 0.297029702970297, "marketing": 0.2532188841201717, "professional_law": 0.2622309197651663, "management": 0.28431372549019607, "college_physics": 0.25742574257425743, "jurisprudence": 0.32710280373831774, "world_religions": 0.31176470588235294, "sociology": 0.275, "us_foreign_policy": 0.21212121212121213, "high_school_macroeconomics": 0.2416452442159383, "computer_security": 0.23232323232323232, "moral_scenarios": 0.2539149888143177, "moral_disputes": 0.3159420289855073, "electrical_engineering": 0.2638888888888889, "astronomy": 0.2582781456953642, "college_biology": 0.2727272727272727 } }, "prompt_2": { "accuracy": 0.2619949946371112, "category_acc": { "high_school_european_history": 0.2804878048780488, "business_ethics": 0.25252525252525254, "clinical_knowledge": 0.3333333333333333, "medical_genetics": 0.26262626262626265, "high_school_us_history": 0.2413793103448276, "high_school_physics": 0.26666666666666666, "high_school_world_history": 0.2245762711864407, "virology": 0.21212121212121213, "high_school_microeconomics": 0.2109704641350211, "econometrics": 0.23008849557522124, "college_computer_science": 0.16161616161616163, "high_school_biology": 0.27184466019417475, "abstract_algebra": 0.31313131313131315, "professional_accounting": 0.3096085409252669, "philosophy": 0.267741935483871, "professional_medicine": 0.2878228782287823, "nutrition": 0.2754098360655738, "global_facts": 0.31313131313131315, "machine_learning": 0.1981981981981982, "security_studies": 0.26229508196721313, "public_relations": 0.26605504587155965, "professional_psychology": 0.24386252045826515, "prehistory": 0.23839009287925697, "anatomy": 0.2835820895522388, "human_sexuality": 0.2692307692307692, "college_medicine": 0.25, "high_school_government_and_politics": 0.328125, "college_chemistry": 0.21212121212121213, "logical_fallacies": 0.2654320987654321, "high_school_geography": 0.3248730964467005, "elementary_mathematics": 0.23607427055702918, "human_aging": 0.16666666666666666, "college_mathematics": 0.2222222222222222, "high_school_psychology": 0.28308823529411764, "formal_logic": 0.184, "high_school_statistics": 0.3116279069767442, "international_law": 0.275, "high_school_mathematics": 0.24535315985130113, "high_school_computer_science": 0.29292929292929293, "conceptual_physics": 0.3076923076923077, "miscellaneous": 0.25831202046035806, "high_school_chemistry": 0.2524752475247525, "marketing": 0.1888412017167382, "professional_law": 0.26157860404435745, "management": 0.3333333333333333, "college_physics": 0.2376237623762376, "jurisprudence": 0.2336448598130841, "world_religions": 0.2529411764705882, "sociology": 0.255, "us_foreign_policy": 0.2828282828282828, "high_school_macroeconomics": 0.2802056555269923, "computer_security": 0.2828282828282828, "moral_scenarios": 0.2505592841163311, "moral_disputes": 0.2927536231884058, "electrical_engineering": 0.2847222222222222, "astronomy": 0.31788079470198677, "college_biology": 0.26573426573426573 } }, "prompt_3": { "accuracy": 0.2679299249195567, "category_acc": { "high_school_european_history": 0.2804878048780488, "business_ethics": 0.2727272727272727, "clinical_knowledge": 0.2916666666666667, "medical_genetics": 0.29292929292929293, "high_school_us_history": 0.2660098522167488, "high_school_physics": 0.34, "high_school_world_history": 0.2669491525423729, "virology": 0.20606060606060606, "high_school_microeconomics": 0.25738396624472576, "econometrics": 0.23893805309734514, "college_computer_science": 0.23232323232323232, "high_school_biology": 0.284789644012945, "abstract_algebra": 0.32323232323232326, "professional_accounting": 0.2846975088967972, "philosophy": 0.2903225806451613, "professional_medicine": 0.2952029520295203, "nutrition": 0.2721311475409836, "global_facts": 0.2828282828282828, "machine_learning": 0.23423423423423423, "security_studies": 0.22540983606557377, "public_relations": 0.25688073394495414, "professional_psychology": 0.2553191489361702, "prehistory": 0.24458204334365324, "anatomy": 0.3208955223880597, "human_sexuality": 0.23846153846153847, "college_medicine": 0.27325581395348836, "high_school_government_and_politics": 0.3177083333333333, "college_chemistry": 0.23232323232323232, "logical_fallacies": 0.29012345679012347, "high_school_geography": 0.3299492385786802, "elementary_mathematics": 0.2519893899204244, "human_aging": 0.16666666666666666, "college_mathematics": 0.1919191919191919, "high_school_psychology": 0.33639705882352944, "formal_logic": 0.192, "high_school_statistics": 0.2930232558139535, "international_law": 0.21666666666666667, "high_school_mathematics": 0.26765799256505574, "high_school_computer_science": 0.24242424242424243, "conceptual_physics": 0.2564102564102564, "miscellaneous": 0.29028132992327366, "high_school_chemistry": 0.2524752475247525, "marketing": 0.21888412017167383, "professional_law": 0.2576647097195042, "management": 0.3431372549019608, "college_physics": 0.24752475247524752, "jurisprudence": 0.2523364485981308, "world_religions": 0.21764705882352942, "sociology": 0.23, "us_foreign_policy": 0.29292929292929293, "high_school_macroeconomics": 0.2879177377892031, "computer_security": 0.23232323232323232, "moral_scenarios": 0.24384787472035793, "moral_disputes": 0.28405797101449276, "electrical_engineering": 0.3541666666666667, "astronomy": 0.33112582781456956, "college_biology": 0.24475524475524477 } }, "prompt_4": { "accuracy": 0.2679299249195567, "category_acc": { "high_school_european_history": 0.3048780487804878, "business_ethics": 0.2727272727272727, "clinical_knowledge": 0.2689393939393939, "medical_genetics": 0.29292929292929293, "high_school_us_history": 0.2660098522167488, "high_school_physics": 0.30666666666666664, "high_school_world_history": 0.2669491525423729, "virology": 0.20606060606060606, "high_school_microeconomics": 0.2742616033755274, "econometrics": 0.2743362831858407, "college_computer_science": 0.20202020202020202, "high_school_biology": 0.31715210355987056, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.2918149466192171, "philosophy": 0.26129032258064516, "professional_medicine": 0.2915129151291513, "nutrition": 0.3114754098360656, "global_facts": 0.25252525252525254, "machine_learning": 0.23423423423423423, "security_studies": 0.27049180327868855, "public_relations": 0.23853211009174313, "professional_psychology": 0.23240589198036007, "prehistory": 0.26006191950464397, "anatomy": 0.26865671641791045, "human_sexuality": 0.3230769230769231, "college_medicine": 0.28488372093023256, "high_school_government_and_politics": 0.2864583333333333, "college_chemistry": 0.32323232323232326, "logical_fallacies": 0.2716049382716049, "high_school_geography": 0.350253807106599, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.14414414414414414, "college_mathematics": 0.1717171717171717, "high_school_psychology": 0.3235294117647059, "formal_logic": 0.184, "high_school_statistics": 0.26046511627906976, "international_law": 0.25, "high_school_mathematics": 0.31970260223048325, "high_school_computer_science": 0.30303030303030304, "conceptual_physics": 0.27350427350427353, "miscellaneous": 0.2595907928388747, "high_school_chemistry": 0.22772277227722773, "marketing": 0.2017167381974249, "professional_law": 0.2563600782778865, "management": 0.3137254901960784, "college_physics": 0.3465346534653465, "jurisprudence": 0.2803738317757009, "world_religions": 0.2, "sociology": 0.285, "us_foreign_policy": 0.25252525252525254, "high_school_macroeconomics": 0.2647814910025707, "computer_security": 0.21212121212121213, "moral_scenarios": 0.25727069351230425, "moral_disputes": 0.28405797101449276, "electrical_engineering": 0.3125, "astronomy": 0.3443708609271523, "college_biology": 0.2517482517482518 } }, "prompt_5": { "accuracy": 0.2608509116910976, "category_acc": { "high_school_european_history": 0.3048780487804878, "business_ethics": 0.25252525252525254, "clinical_knowledge": 0.26515151515151514, "medical_genetics": 0.25252525252525254, "high_school_us_history": 0.2857142857142857, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.2584745762711864, "virology": 0.23636363636363636, "high_school_microeconomics": 0.25316455696202533, "econometrics": 0.23008849557522124, "college_computer_science": 0.21212121212121213, "high_school_biology": 0.284789644012945, "abstract_algebra": 0.31313131313131315, "professional_accounting": 0.29537366548042704, "philosophy": 0.2709677419354839, "professional_medicine": 0.26199261992619927, "nutrition": 0.2459016393442623, "global_facts": 0.2727272727272727, "machine_learning": 0.21621621621621623, "security_studies": 0.27049180327868855, "public_relations": 0.23853211009174313, "professional_psychology": 0.25368248772504093, "prehistory": 0.24148606811145512, "anatomy": 0.3208955223880597, "human_sexuality": 0.2230769230769231, "college_medicine": 0.2558139534883721, "high_school_government_and_politics": 0.2916666666666667, "college_chemistry": 0.2222222222222222, "logical_fallacies": 0.32098765432098764, "high_school_geography": 0.29949238578680204, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.16216216216216217, "college_mathematics": 0.21212121212121213, "high_school_psychology": 0.29044117647058826, "formal_logic": 0.176, "high_school_statistics": 0.26046511627906976, "international_law": 0.275, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.2727272727272727, "conceptual_physics": 0.26495726495726496, "miscellaneous": 0.2659846547314578, "high_school_chemistry": 0.30198019801980197, "marketing": 0.21888412017167383, "professional_law": 0.253750815394651, "management": 0.29411764705882354, "college_physics": 0.2376237623762376, "jurisprudence": 0.24299065420560748, "world_religions": 0.2411764705882353, "sociology": 0.255, "us_foreign_policy": 0.25252525252525254, "high_school_macroeconomics": 0.2647814910025707, "computer_security": 0.2727272727272727, "moral_scenarios": 0.24720357941834453, "moral_disputes": 0.2782608695652174, "electrical_engineering": 0.2847222222222222, "astronomy": 0.304635761589404, "college_biology": 0.24475524475524477 } } }, "c_eval": { "prompt_1": { "accuracy": 0.25705794947994054 }, "prompt_2": { "accuracy": 0.2540861812778603 }, "prompt_3": { "accuracy": 0.2600297176820208 }, "prompt_4": { "accuracy": 0.25482912332838037 }, "prompt_5": { "accuracy": 0.2451708766716196 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.27023661270236615, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.375, "computer_architecture": 0.15384615384615385, "college_programming": 0.2619047619047619, "college_physics": 0.2916666666666667, "college_chemistry": 0.4827586206896552, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.27586206896551724, "high_school_mathematics": 0.43478260869565216, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.041666666666666664, "middle_school_mathematics": 0.25, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.25, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.18333333333333332, "business_administration": 0.21052631578947367, "marxism": 0.25, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.20588235294117646, "teacher_qualification": 0.32653061224489793, "high_school_politics": 0.08333333333333333, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.5882352941176471, "modern_chinese_history": 0.32142857142857145, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.13793103448275862, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.2894736842105263, "professional_tour_guide": 0.23529411764705882, "legal_professional": 0.17857142857142858, "high_school_chinese": 0.16666666666666666, "high_school_history": 0.32, "middle_school_history": 0.2962962962962963, "civil_servant": 0.21153846153846154, "sports_science": 0.3333333333333333, "plant_protection": 0.14814814814814814, "basic_medicine": 0.5833333333333334, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.2222222222222222, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.1388888888888889, "tax_accountant": 0.2962962962962963, "physician": 0.2777777777777778 } }, "prompt_2": { "accuracy": 0.28393524283935245, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.375, "computer_architecture": 0.15384615384615385, "college_programming": 0.16666666666666666, "college_physics": 0.25, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.375, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3103448275862069, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.16666666666666666, "middle_school_mathematics": 0.375, "middle_school_biology": 0.19230769230769232, "middle_school_physics": 0.25, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.25, "college_economics": 0.23333333333333334, "business_administration": 0.23684210526315788, "marxism": 0.375, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.2647058823529412, "teacher_qualification": 0.3469387755102041, "high_school_politics": 0.08333333333333333, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.20833333333333334, "logic": 0.4444444444444444, "law": 0.27586206896551724, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.21052631578947367, "professional_tour_guide": 0.20588235294117646, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.28, "middle_school_history": 0.3333333333333333, "civil_servant": 0.19230769230769232, "sports_science": 0.16666666666666666, "plant_protection": 0.25925925925925924, "basic_medicine": 0.5, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.3137254901960784, "accountant": 0.2777777777777778, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.25925925925925924, "physician": 0.2777777777777778 } }, "prompt_3": { "accuracy": 0.2714819427148194, "category_acc": { "computer_network": 0.25, "operating_system": 0.3333333333333333, "computer_architecture": 0.19230769230769232, "college_programming": 0.19047619047619047, "college_physics": 0.25, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.5217391304347826, "high_school_physics": 0.25, "high_school_chemistry": 0.375, "high_school_biology": 0.125, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.3076923076923077, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.25, "college_economics": 0.2, "business_administration": 0.2894736842105263, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.17647058823529413, "teacher_qualification": 0.42857142857142855, "high_school_politics": 0.125, "high_school_geography": 0.25, "middle_school_politics": 0.15384615384615385, "middle_school_geography": 0.5294117647058824, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.08333333333333333, "logic": 0.3333333333333333, "law": 0.3103448275862069, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.23529411764705882, "legal_professional": 0.14285714285714285, "high_school_chinese": 0.125, "high_school_history": 0.4, "middle_school_history": 0.25925925925925924, "civil_servant": 0.19230769230769232, "sports_science": 0.2916666666666667, "plant_protection": 0.25925925925925924, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.2962962962962963, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.2962962962962963, "physician": 0.25925925925925924 } }, "prompt_4": { "accuracy": 0.26899128268991285, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.375, "computer_architecture": 0.23076923076923078, "college_programming": 0.2857142857142857, "college_physics": 0.20833333333333334, "college_chemistry": 0.41379310344827586, "advanced_mathematics": 0.125, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.47619047619047616, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.27586206896551724, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.25, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.38461538461538464, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.16, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.23333333333333334, "business_administration": 0.39473684210526316, "marxism": 0.25, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.20833333333333334, "high_school_geography": 0.25, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.32142857142857145, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.3103448275862069, "chinese_language_and_literature": 0.25, "art_studies": 0.21052631578947367, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.14285714285714285, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.36, "middle_school_history": 0.2222222222222222, "civil_servant": 0.25, "sports_science": 0.375, "plant_protection": 0.14814814814814814, "basic_medicine": 0.4583333333333333, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.19607843137254902, "accountant": 0.2037037037037037, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.3333333333333333, "tax_accountant": 0.25925925925925924, "physician": 0.3333333333333333 } }, "prompt_5": { "accuracy": 0.2627646326276463, "category_acc": { "computer_network": 0.16666666666666666, "operating_system": 0.3333333333333333, "computer_architecture": 0.23076923076923078, "college_programming": 0.19047619047619047, "college_physics": 0.16666666666666666, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.4166666666666667, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.375, "middle_school_biology": 0.3076923076923077, "middle_school_physics": 0.20833333333333334, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.25, "business_administration": 0.42105263157894735, "marxism": 0.25, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.20588235294117646, "teacher_qualification": 0.30612244897959184, "high_school_politics": 0.125, "high_school_geography": 0.25, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.25925925925925924, "law": 0.2413793103448276, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.17647058823529413, "legal_professional": 0.14285714285714285, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.36, "middle_school_history": 0.3333333333333333, "civil_servant": 0.2692307692307692, "sports_science": 0.16666666666666666, "plant_protection": 0.2222222222222222, "basic_medicine": 0.4583333333333333, "clinical_medicine": 0.14814814814814814, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.2037037037037037, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.2777777777777778, "physician": 0.24074074074074073 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.2222222222222222 }, "prompt_2": { "accuracy": 0.2867383512544803 }, "prompt_3": { "accuracy": 0.24372759856630824 }, "prompt_4": { "accuracy": 0.23655913978494625 }, "prompt_5": { "accuracy": 0.25806451612903225 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2523743740286652, "category_acc": { "agronomy": 0.25443786982248523, "anatomy": 0.28378378378378377, "ancient_chinese": 0.2804878048780488, "arts": 0.26875, "astronomy": 0.30303030303030304, "business_ethics": 0.22966507177033493, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.2366412213740458, "chinese_food_culture": 0.29411764705882354, "chinese_foreign_policy": 0.21495327102803738, "chinese_history": 0.23219814241486067, "chinese_literature": 0.23529411764705882, "chinese_teacher_qualification": 0.22905027932960895, "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.2358490566037736, "college_education": 0.24299065420560748, "college_engineering_hydrology": 0.330188679245283, "college_law": 0.3425925925925926, "college_mathematics": 0.3238095238095238, "college_medical_statistics": 0.19811320754716982, "college_medicine": 0.23809523809523808, "computer_science": 0.24509803921568626, "computer_security": 0.22807017543859648, "conceptual_physics": 0.25170068027210885, "construction_project_management": 0.2517985611510791, "economics": 0.22641509433962265, "education": 0.3067484662576687, "electrical_engineering": 0.20348837209302326, "elementary_chinese": 0.2698412698412698, "elementary_commonsense": 0.22727272727272727, "elementary_information_and_technology": 0.20588235294117646, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.22962962962962963, "food_science": 0.22377622377622378, "genetics": 0.2727272727272727, "global_facts": 0.3087248322147651, "high_school_biology": 0.2603550295857988, "high_school_chemistry": 0.22727272727272727, "high_school_geography": 0.2627118644067797, "high_school_mathematics": 0.21341463414634146, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.36363636363636365, "human_sexuality": 0.30158730158730157, "international_law": 0.23783783783783785, "journalism": 0.2616279069767442, "jurisprudence": 0.2384428223844282, "legal_and_moral_basis": 0.22429906542056074, "logical": 0.2764227642276423, "machine_learning": 0.21311475409836064, "management": 0.24285714285714285, "marketing": 0.2611111111111111, "marxist_theory": 0.26455026455026454, "modern_chinese": 0.28448275862068967, "nutrition": 0.2827586206896552, "philosophy": 0.34285714285714286, "professional_accounting": 0.2571428571428571, "professional_law": 0.21800947867298578, "professional_medicine": 0.2393617021276596, "professional_psychology": 0.24568965517241378, "public_relations": 0.27011494252873564, "security_study": 0.2222222222222222, "sociology": 0.21238938053097345, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.2781065088757396, "world_history": 0.2608695652173913, "world_religions": 0.25 } }, "prompt_2": { "accuracy": 0.2562597133482991, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.28378378378378377, "ancient_chinese": 0.24390243902439024, "arts": 0.3125, "astronomy": 0.23636363636363636, "business_ethics": 0.23923444976076555, "chinese_civil_service_exam": 0.26875, "chinese_driving_rule": 0.21374045801526717, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.2616822429906542, "chinese_history": 0.25386996904024767, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.2681564245810056, "clinical_knowledge": 0.2742616033755274, "college_actuarial_science": 0.18867924528301888, "college_education": 0.18691588785046728, "college_engineering_hydrology": 0.3113207547169811, "college_law": 0.32407407407407407, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.19811320754716982, "college_medicine": 0.2454212454212454, "computer_science": 0.27941176470588236, "computer_security": 0.21052631578947367, "conceptual_physics": 0.25170068027210885, "construction_project_management": 0.26618705035971224, "economics": 0.2389937106918239, "education": 0.24539877300613497, "electrical_engineering": 0.20348837209302326, "elementary_chinese": 0.23015873015873015, "elementary_commonsense": 0.24242424242424243, "elementary_information_and_technology": 0.23109243697478993, "elementary_mathematics": 0.24782608695652175, "ethnology": 0.31851851851851853, "food_science": 0.27972027972027974, "genetics": 0.26136363636363635, "global_facts": 0.26174496644295303, "high_school_biology": 0.21301775147928995, "high_school_chemistry": 0.25, "high_school_geography": 0.2966101694915254, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.23636363636363636, "high_school_politics": 0.36363636363636365, "human_sexuality": 0.2857142857142857, "international_law": 0.20540540540540542, "journalism": 0.26744186046511625, "jurisprudence": 0.2384428223844282, "legal_and_moral_basis": 0.26635514018691586, "logical": 0.25203252032520324, "machine_learning": 0.23770491803278687, "management": 0.24285714285714285, "marketing": 0.3, "marxist_theory": 0.25925925925925924, "modern_chinese": 0.2672413793103448, "nutrition": 0.27586206896551724, "philosophy": 0.29523809523809524, "professional_accounting": 0.29714285714285715, "professional_law": 0.21800947867298578, "professional_medicine": 0.3058510638297872, "professional_psychology": 0.27155172413793105, "public_relations": 0.22413793103448276, "security_study": 0.25925925925925924, "sociology": 0.2610619469026549, "sports_science": 0.2606060606060606, "traditional_chinese_medicine": 0.23783783783783785, "virology": 0.2603550295857988, "world_history": 0.2919254658385093, "world_religions": 0.225 } }, "prompt_3": { "accuracy": 0.2538421688827491, "category_acc": { "agronomy": 0.20710059171597633, "anatomy": 0.31756756756756754, "ancient_chinese": 0.23170731707317074, "arts": 0.2875, "astronomy": 0.24242424242424243, "business_ethics": 0.22488038277511962, "chinese_civil_service_exam": 0.21875, "chinese_driving_rule": 0.2366412213740458, "chinese_food_culture": 0.23529411764705882, "chinese_foreign_policy": 0.2523364485981308, "chinese_history": 0.24148606811145512, "chinese_literature": 0.27450980392156865, "chinese_teacher_qualification": 0.26256983240223464, "clinical_knowledge": 0.22784810126582278, "college_actuarial_science": 0.24528301886792453, "college_education": 0.22429906542056074, "college_engineering_hydrology": 0.3490566037735849, "college_law": 0.3148148148148148, "college_mathematics": 0.3142857142857143, "college_medical_statistics": 0.19811320754716982, "college_medicine": 0.27472527472527475, "computer_science": 0.24509803921568626, "computer_security": 0.29239766081871343, "conceptual_physics": 0.2653061224489796, "construction_project_management": 0.2158273381294964, "economics": 0.2138364779874214, "education": 0.2392638036809816, "electrical_engineering": 0.22674418604651161, "elementary_chinese": 0.24603174603174602, "elementary_commonsense": 0.23737373737373738, "elementary_information_and_technology": 0.23949579831932774, "elementary_mathematics": 0.27391304347826084, "ethnology": 0.21481481481481482, "food_science": 0.2727272727272727, "genetics": 0.30113636363636365, "global_facts": 0.28859060402684567, "high_school_biology": 0.2781065088757396, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.2682926829268293, "high_school_physics": 0.2545454545454545, "high_school_politics": 0.27972027972027974, "human_sexuality": 0.23809523809523808, "international_law": 0.24864864864864866, "journalism": 0.26744186046511625, "jurisprudence": 0.24087591240875914, "legal_and_moral_basis": 0.21962616822429906, "logical": 0.3170731707317073, "machine_learning": 0.20491803278688525, "management": 0.26666666666666666, "marketing": 0.2111111111111111, "marxist_theory": 0.24867724867724866, "modern_chinese": 0.23275862068965517, "nutrition": 0.30344827586206896, "philosophy": 0.3047619047619048, "professional_accounting": 0.26857142857142857, "professional_law": 0.2559241706161137, "professional_medicine": 0.26063829787234044, "professional_psychology": 0.23275862068965517, "public_relations": 0.25287356321839083, "security_study": 0.25925925925925924, "sociology": 0.24778761061946902, "sports_science": 0.2909090909090909, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.23076923076923078, "world_history": 0.2795031055900621, "world_religions": 0.2125 } }, "prompt_4": { "accuracy": 0.25962700742531514, "category_acc": { "agronomy": 0.2781065088757396, "anatomy": 0.25675675675675674, "ancient_chinese": 0.2804878048780488, "arts": 0.3, "astronomy": 0.23636363636363636, "business_ethics": 0.22009569377990432, "chinese_civil_service_exam": 0.2375, "chinese_driving_rule": 0.2366412213740458, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.2616822429906542, "chinese_history": 0.24458204334365324, "chinese_literature": 0.24019607843137256, "chinese_teacher_qualification": 0.26256983240223464, "clinical_knowledge": 0.21518987341772153, "college_actuarial_science": 0.25471698113207547, "college_education": 0.22429906542056074, "college_engineering_hydrology": 0.330188679245283, "college_law": 0.3148148148148148, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.30036630036630035, "computer_science": 0.2549019607843137, "computer_security": 0.28654970760233917, "conceptual_physics": 0.19047619047619047, "construction_project_management": 0.26618705035971224, "economics": 0.1761006289308176, "education": 0.25153374233128833, "electrical_engineering": 0.1744186046511628, "elementary_chinese": 0.2976190476190476, "elementary_commonsense": 0.23232323232323232, "elementary_information_and_technology": 0.2647058823529412, "elementary_mathematics": 0.2782608695652174, "ethnology": 0.23703703703703705, "food_science": 0.24475524475524477, "genetics": 0.26704545454545453, "global_facts": 0.2751677852348993, "high_school_biology": 0.26627218934911245, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.3220338983050847, "high_school_mathematics": 0.25, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.3076923076923077, "human_sexuality": 0.2857142857142857, "international_law": 0.24324324324324326, "journalism": 0.28488372093023256, "jurisprudence": 0.2749391727493917, "legal_and_moral_basis": 0.2616822429906542, "logical": 0.2682926829268293, "machine_learning": 0.2786885245901639, "management": 0.2619047619047619, "marketing": 0.26666666666666666, "marxist_theory": 0.2698412698412698, "modern_chinese": 0.2413793103448276, "nutrition": 0.27586206896551724, "philosophy": 0.2857142857142857, "professional_accounting": 0.26857142857142857, "professional_law": 0.27014218009478674, "professional_medicine": 0.2473404255319149, "professional_psychology": 0.22413793103448276, "public_relations": 0.26436781609195403, "security_study": 0.18518518518518517, "sociology": 0.23008849557522124, "sports_science": 0.3333333333333333, "traditional_chinese_medicine": 0.23783783783783785, "virology": 0.27218934911242604, "world_history": 0.2857142857142857, "world_religions": 0.275 } }, "prompt_5": { "accuracy": 0.25461923674667586, "category_acc": { "agronomy": 0.28402366863905326, "anatomy": 0.2972972972972973, "ancient_chinese": 0.2621951219512195, "arts": 0.25625, "astronomy": 0.20606060606060606, "business_ethics": 0.24880382775119617, "chinese_civil_service_exam": 0.275, "chinese_driving_rule": 0.2366412213740458, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.2523364485981308, "chinese_history": 0.23219814241486067, "chinese_literature": 0.23039215686274508, "chinese_teacher_qualification": 0.2346368715083799, "clinical_knowledge": 0.24050632911392406, "college_actuarial_science": 0.2169811320754717, "college_education": 0.19626168224299065, "college_engineering_hydrology": 0.2830188679245283, "college_law": 0.37962962962962965, "college_mathematics": 0.24761904761904763, "college_medical_statistics": 0.19811320754716982, "college_medicine": 0.23809523809523808, "computer_science": 0.24509803921568626, "computer_security": 0.2573099415204678, "conceptual_physics": 0.2857142857142857, "construction_project_management": 0.2446043165467626, "economics": 0.2389937106918239, "education": 0.26993865030674846, "electrical_engineering": 0.21511627906976744, "elementary_chinese": 0.26587301587301587, "elementary_commonsense": 0.23232323232323232, "elementary_information_and_technology": 0.24369747899159663, "elementary_mathematics": 0.20434782608695654, "ethnology": 0.2518518518518518, "food_science": 0.25874125874125875, "genetics": 0.26136363636363635, "global_facts": 0.3288590604026846, "high_school_biology": 0.3254437869822485, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.3050847457627119, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.21818181818181817, "high_school_politics": 0.3076923076923077, "human_sexuality": 0.30952380952380953, "international_law": 0.2702702702702703, "journalism": 0.20930232558139536, "jurisprudence": 0.2384428223844282, "legal_and_moral_basis": 0.24766355140186916, "logical": 0.3333333333333333, "machine_learning": 0.2540983606557377, "management": 0.2714285714285714, "marketing": 0.3111111111111111, "marxist_theory": 0.2804232804232804, "modern_chinese": 0.29310344827586204, "nutrition": 0.22758620689655173, "philosophy": 0.29523809523809524, "professional_accounting": 0.25142857142857145, "professional_law": 0.22748815165876776, "professional_medicine": 0.2579787234042553, "professional_psychology": 0.2025862068965517, "public_relations": 0.2471264367816092, "security_study": 0.2518518518518518, "sociology": 0.23008849557522124, "sports_science": 0.32727272727272727, "traditional_chinese_medicine": 0.21621621621621623, "virology": 0.2485207100591716, "world_history": 0.22981366459627328, "world_religions": 0.25625 } } }, "zbench": { "prompt_1": { "accuracy": 0.3333333333333333 }, "prompt_2": { "accuracy": 0.3333333333333333 }, "prompt_3": { "accuracy": 0.3333333333333333 }, "prompt_4": { "accuracy": 0.36363636363636365 }, "prompt_5": { "accuracy": 0.21212121212121213 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.225 }, "prompt_2": { "accuracy": 0.24772727272727274 }, "prompt_3": { "accuracy": 0.22272727272727272 }, "prompt_4": { "accuracy": 0.2 }, "prompt_5": { "accuracy": 0.20227272727272727 } }, "ocnli": { "prompt_1": { "accuracy": 0.32949152542372884 }, "prompt_2": { "accuracy": 0.3488135593220339 }, "prompt_3": { "accuracy": 0.32033898305084746 }, "prompt_4": { "accuracy": 0.3416949152542373 }, "prompt_5": { "accuracy": 0.3396610169491525 } }, "c3": { "prompt_1": { "accuracy": 0.3051608077786088 }, "prompt_2": { "accuracy": 0.30254300673148843 }, "prompt_3": { "accuracy": 0.31787584143605085 }, "prompt_4": { "accuracy": 0.3070306656694091 }, "prompt_5": { "accuracy": 0.31787584143605085 } }, "dream": { "prompt_1": { "accuracy": 0.37236648701616853 }, "prompt_2": { "accuracy": 0.37677609015188634 }, "prompt_3": { "accuracy": 0.37677609015188634 }, "prompt_4": { "accuracy": 0.3728564429201372 }, "prompt_5": { "accuracy": 0.36893679568838805 } }, "samsum": { "prompt_1": { "rouge1": 0.2850073990198992, "rouge2": 0.08352220743518561, "rougeL": 0.21206530932751466, "avg_rouge": 0.19353163859419983 }, "prompt_2": { "rouge1": 0.30658379785153117, "rouge2": 0.09320374513029613, "rougeL": 0.22341544203058414, "avg_rouge": 0.2077343283374705 }, "prompt_3": { "rouge1": 0.28776691352486655, "rouge2": 0.08167280160298225, "rougeL": 0.20838776666273462, "avg_rouge": 0.19260916059686115 }, "prompt_4": { "rouge1": 0.2756527589777085, "rouge2": 0.08054095290677721, "rougeL": 0.20134716056134175, "avg_rouge": 0.1858469574819425 }, "prompt_5": { "rouge1": 0.3053455922803706, "rouge2": 0.0909102250678557, "rougeL": 0.22428544015406174, "avg_rouge": 0.20684708583409603 } }, "dialogsum": { "prompt_1": { "rouge1": 0.22417868322315565, "rouge2": 0.06099633594873171, "rougeL": 0.1645146116712034, "avg_rouge": 0.14989654361436358 }, "prompt_2": { "rouge1": 0.2196654150670897, "rouge2": 0.06032763846133467, "rougeL": 0.16056995602370785, "avg_rouge": 0.14685433651737742 }, "prompt_3": { "rouge1": 0.21832150041066453, "rouge2": 0.059156306603515486, "rougeL": 0.16139471401530808, "avg_rouge": 0.1462908403431627 }, "prompt_4": { "rouge1": 0.21922502072906003, "rouge2": 0.05878269252032803, "rougeL": 0.1592118241412318, "avg_rouge": 0.1457398457968733 }, "prompt_5": { "rouge1": 0.21060609015887038, "rouge2": 0.057833836582119554, "rougeL": 0.15643443484914793, "avg_rouge": 0.14162478719671262 } }, "sst2": { "prompt_1": { "accuracy": 0.6731651376146789 }, "prompt_2": { "accuracy": 0.5756880733944955 }, "prompt_3": { "accuracy": 0.6043577981651376 }, "prompt_4": { "accuracy": 0.5665137614678899 }, "prompt_5": { "accuracy": 0.6903669724770642 } }, "cola": { "prompt_1": { "accuracy": 0.5244487056567594 }, "prompt_2": { "accuracy": 0.5225311601150527 }, "prompt_3": { "accuracy": 0.5445829338446788 }, "prompt_4": { "accuracy": 0.6442953020134228 }, "prompt_5": { "accuracy": 0.5349952061361457 } }, "qqp": { "prompt_1": { "accuracy": 0.4965 }, "prompt_2": { "accuracy": 0.498 }, "prompt_3": { "accuracy": 0.495 }, "prompt_4": { "accuracy": 0.5025 }, "prompt_5": { "accuracy": 0.496 } }, "mnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.5 } }, "qnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.5 } }, "mrpc": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.6 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "sailor_1_8b_chat": { "model_size": "1.8B", "model_link": "https://huggingface.co/sail/Sailor-1.8B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.6323529411764706, "language_acc": { "Spanish": 0.6109243697478992, "English": 0.6672268907563025, "Chinese": 0.6386554621848739, "Vietnamese": 0.6126050420168068 }, "consistency_score_2": 0.6215686274509804, "consistency_score_3": 0.4714285714285714, "consistency_score_4": 0.37815126050420167, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.6546218487394958, "Spanish,Chinese": 0.6025210084033613, "Spanish,Vietnamese": 0.6210084033613446, "English,Chinese": 0.6235294117647059, "English,Vietnamese": 0.6369747899159663, "Chinese,Vietnamese": 0.5907563025210084 }, "3_combine": { "Spanish,English,Chinese": 0.47983193277310926, "Spanish,English,Vietnamese": 0.4899159663865546, "Spanish,Chinese,Vietnamese": 0.4504201680672269, "English,Chinese,Vietnamese": 0.46554621848739497 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.37815126050420167 } }, "AC3_2": 0.6269144091851598, "AC3_3": 0.5401598781391099, "AC3_4": 0.4732787085259903 }, "prompt_2": { "overall_acc": 0.6466386554621849, "language_acc": { "Spanish": 0.6571428571428571, "English": 0.6663865546218487, "Chinese": 0.6504201680672269, "Vietnamese": 0.6126050420168068 }, "consistency_score_2": 0.6168067226890757, "consistency_score_3": 0.47016806722689075, "consistency_score_4": 0.38235294117647056, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.66890756302521, "Spanish,Chinese": 0.5949579831932773, "Spanish,Vietnamese": 0.6184873949579832, "English,Chinese": 0.5974789915966386, "English,Vietnamese": 0.6378151260504201, "Chinese,Vietnamese": 0.5831932773109244 }, "3_combine": { "Spanish,English,Chinese": 0.473109243697479, "Spanish,English,Vietnamese": 0.5033613445378151, "Spanish,Chinese,Vietnamese": 0.4504201680672269, "English,Chinese,Vietnamese": 0.453781512605042 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.38235294117647056 } }, "AC3_2": 0.6313704995965111, "AC3_3": 0.5444609897647811, "AC3_4": 0.48055628943519973 }, "prompt_3": { "overall_acc": 0.6630252100840336, "language_acc": { "Spanish": 0.6747899159663866, "English": 0.7025210084033613, "Chinese": 0.634453781512605, "Vietnamese": 0.6403361344537815 }, "consistency_score_2": 0.6438375350140056, "consistency_score_3": 0.5039915966386554, "consistency_score_4": 0.42016806722689076, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7151260504201681, "Spanish,Chinese": 0.6201680672268908, "Spanish,Vietnamese": 0.6655462184873949, "English,Chinese": 0.6025210084033613, "English,Vietnamese": 0.6512605042016807, "Chinese,Vietnamese": 0.6084033613445378 }, "3_combine": { "Spanish,English,Chinese": 0.5058823529411764, "Spanish,English,Vietnamese": 0.5445378151260504, "Spanish,Chinese,Vietnamese": 0.4865546218487395, "English,Chinese,Vietnamese": 0.4789915966386555 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.42016806722689076 } }, "AC3_2": 0.6532905134547375, "AC3_3": 0.5726723596238706, "AC3_4": 0.5143717688307397 }, "prompt_4": { "overall_acc": 0.6373949579831933, "language_acc": { "Spanish": 0.6411764705882353, "English": 0.6823529411764706, "Chinese": 0.6277310924369748, "Vietnamese": 0.5983193277310924 }, "consistency_score_2": 0.6127450980392157, "consistency_score_3": 0.4607142857142857, "consistency_score_4": 0.3663865546218487, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.6697478991596638, "Spanish,Chinese": 0.6033613445378151, "Spanish,Vietnamese": 0.6159663865546219, "English,Chinese": 0.6050420168067226, "English,Vietnamese": 0.6142857142857143, "Chinese,Vietnamese": 0.5680672268907563 }, "3_combine": { "Spanish,English,Chinese": 0.4756302521008403, "Spanish,English,Vietnamese": 0.4890756302521008, "Spanish,Chinese,Vietnamese": 0.4369747899159664, "English,Chinese,Vietnamese": 0.4411764705882353 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3663865546218487 } }, "AC3_2": 0.6248270089521416, "AC3_3": 0.534841072404764, "AC3_4": 0.46530632340391215 }, "prompt_5": { "overall_acc": 0.6640756302521008, "language_acc": { "Spanish": 0.6697478991596638, "English": 0.6966386554621848, "Chinese": 0.6630252100840336, "Vietnamese": 0.626890756302521 }, "consistency_score_2": 0.6662464985994397, "consistency_score_3": 0.5294117647058824, "consistency_score_4": 0.44537815126050423, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7151260504201681, "Spanish,Chinese": 0.6512605042016807, "Spanish,Vietnamese": 0.6899159663865546, "English,Chinese": 0.646218487394958, "English,Vietnamese": 0.6840336134453782, "Chinese,Vietnamese": 0.6109243697478992 }, "3_combine": { "Spanish,English,Chinese": 0.5369747899159664, "Spanish,English,Vietnamese": 0.5672268907563025, "Spanish,Chinese,Vietnamese": 0.5100840336134453, "English,Chinese,Vietnamese": 0.5033613445378151 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.44537815126050423 } }, "AC3_2": 0.6651592931245526, "AC3_3": 0.589146484103178, "AC3_4": 0.5331718750265324 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.3647619047619047, "language_acc": { "Filipino": 0.23333333333333334, "Vietnamese": 0.35333333333333333, "Chinese": 0.38, "Spanish": 0.36666666666666664, "Malay": 0.3933333333333333, "Indonesian": 0.41333333333333333, "English": 0.41333333333333333 }, "consistency_score_2": 0.45968253968253964, "consistency_score_3": 0.2721904761904761, "consistency_score_4": 0.18666666666666668, "consistency_score_5": 0.13999999999999999, "consistency_score_6": 0.10952380952380951, "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.49333333333333335, "Filipino,Chinese": 0.4, "Filipino,Spanish": 0.4266666666666667, "Filipino,Malay": 0.43333333333333335, "Filipino,Indonesian": 0.4266666666666667, "Filipino,English": 0.38666666666666666, "Vietnamese,Chinese": 0.48, "Vietnamese,Spanish": 0.48, "Vietnamese,Malay": 0.4066666666666667, "Vietnamese,Indonesian": 0.48, "Vietnamese,English": 0.4866666666666667, "Chinese,Spanish": 0.43333333333333335, "Chinese,Malay": 0.41333333333333333, "Chinese,Indonesian": 0.44666666666666666, "Chinese,English": 0.44, "Spanish,Malay": 0.44, "Spanish,Indonesian": 0.46, "Spanish,English": 0.49333333333333335, "Malay,Indonesian": 0.5866666666666667, "Malay,English": 0.52, "Indonesian,English": 0.52 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.25333333333333335, "Filipino,Vietnamese,Spanish": 0.2733333333333333, "Filipino,Vietnamese,Malay": 0.25333333333333335, "Filipino,Vietnamese,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,English": 0.26, "Filipino,Chinese,Spanish": 0.21333333333333335, "Filipino,Chinese,Malay": 0.22666666666666666, "Filipino,Chinese,Indonesian": 0.22, "Filipino,Chinese,English": 0.20666666666666667, "Filipino,Spanish,Malay": 0.26666666666666666, "Filipino,Spanish,Indonesian": 0.24, "Filipino,Spanish,English": 0.25333333333333335, "Filipino,Malay,Indonesian": 0.2866666666666667, "Filipino,Malay,English": 0.26, "Filipino,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish": 0.2866666666666667, "Vietnamese,Chinese,Malay": 0.23333333333333334, "Vietnamese,Chinese,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,English": 0.29333333333333333, "Vietnamese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Spanish,Indonesian": 0.3, "Vietnamese,Spanish,English": 0.30666666666666664, "Vietnamese,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Malay,English": 0.2866666666666667, "Vietnamese,Indonesian,English": 0.32, "Chinese,Spanish,Malay": 0.25333333333333335, "Chinese,Spanish,Indonesian": 0.25333333333333335, "Chinese,Spanish,English": 0.25333333333333335, "Chinese,Malay,Indonesian": 0.30666666666666664, "Chinese,Malay,English": 0.2733333333333333, "Chinese,Indonesian,English": 0.2733333333333333, "Spanish,Malay,Indonesian": 0.30666666666666664, "Spanish,Malay,English": 0.28, "Spanish,Indonesian,English": 0.30666666666666664, "Malay,Indonesian,English": 0.37333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.14, "Filipino,Vietnamese,Chinese,Malay": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,English": 0.18, "Filipino,Vietnamese,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay": 0.16, "Filipino,Chinese,Spanish,Indonesian": 0.14, "Filipino,Chinese,Spanish,English": 0.14, "Filipino,Chinese,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Malay,English": 0.15333333333333332, "Filipino,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Spanish,Malay,English": 0.18666666666666668, "Filipino,Spanish,Indonesian,English": 0.18, "Filipino,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,Spanish,English": 0.18666666666666668, "Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Malay,English": 0.18, "Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.22, "Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Chinese,Spanish,Malay,English": 0.18, "Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Chinese,Malay,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian,English": 0.22 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Malay,English": 0.12, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.14, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.16666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, "AC3_2": 0.4067579807710232, "AC3_3": 0.3117492594634734, "AC3_4": 0.246954519241341, "AC3_5": 0.20233962260142327, "AC3_6": 0.16846433348905487, "AC3_7": 0.14005625875941088 }, "prompt_2": { "overall_acc": 0.33714285714285713, "language_acc": { "Filipino": 0.22666666666666666, "Vietnamese": 0.32666666666666666, "Chinese": 0.35333333333333333, "Spanish": 0.32666666666666666, "Malay": 0.37333333333333335, "Indonesian": 0.36666666666666664, "English": 0.38666666666666666 }, "consistency_score_2": 0.4333333333333334, "consistency_score_3": 0.24723809523809526, "consistency_score_4": 0.16971428571428573, "consistency_score_5": 0.1304761904761905, "consistency_score_6": 0.10571428571428573, "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4, "Filipino,Chinese": 0.4, "Filipino,Spanish": 0.4266666666666667, "Filipino,Malay": 0.36666666666666664, "Filipino,Indonesian": 0.38666666666666666, "Filipino,English": 0.38666666666666666, "Vietnamese,Chinese": 0.4266666666666667, "Vietnamese,Spanish": 0.42, "Vietnamese,Malay": 0.5, "Vietnamese,Indonesian": 0.48, "Vietnamese,English": 0.4666666666666667, "Chinese,Spanish": 0.44666666666666666, "Chinese,Malay": 0.42, "Chinese,Indonesian": 0.35333333333333333, "Chinese,English": 0.5066666666666667, "Spanish,Malay": 0.4266666666666667, "Spanish,Indonesian": 0.43333333333333335, "Spanish,English": 0.3933333333333333, "Malay,Indonesian": 0.5133333333333333, "Malay,English": 0.48, "Indonesian,English": 0.4666666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.21333333333333335, "Filipino,Vietnamese,Spanish": 0.24, "Filipino,Vietnamese,Malay": 0.21333333333333335, "Filipino,Vietnamese,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,English": 0.21333333333333335, "Filipino,Chinese,Spanish": 0.24, "Filipino,Chinese,Malay": 0.2, "Filipino,Chinese,Indonesian": 0.2, "Filipino,Chinese,English": 0.23333333333333334, "Filipino,Spanish,Malay": 0.23333333333333334, "Filipino,Spanish,Indonesian": 0.24666666666666667, "Filipino,Spanish,English": 0.22666666666666666, "Filipino,Malay,Indonesian": 0.23333333333333334, "Filipino,Malay,English": 0.2, "Filipino,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish": 0.2733333333333333, "Vietnamese,Chinese,Malay": 0.26666666666666666, "Vietnamese,Chinese,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,English": 0.2866666666666667, "Vietnamese,Spanish,Malay": 0.26666666666666666, "Vietnamese,Spanish,Indonesian": 0.26, "Vietnamese,Spanish,English": 0.23333333333333334, "Vietnamese,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Malay,English": 0.3, "Vietnamese,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay": 0.25333333333333335, "Chinese,Spanish,Indonesian": 0.22, "Chinese,Spanish,English": 0.2733333333333333, "Chinese,Malay,Indonesian": 0.25333333333333335, "Chinese,Malay,English": 0.2733333333333333, "Chinese,Indonesian,English": 0.25333333333333335, "Spanish,Malay,Indonesian": 0.28, "Spanish,Malay,English": 0.22666666666666666, "Spanish,Indonesian,English": 0.23333333333333334, "Malay,Indonesian,English": 0.3 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18, "Filipino,Vietnamese,Chinese,Malay": 0.14, "Filipino,Vietnamese,Chinese,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Spanish,English": 0.15333333333333332, "Filipino,Vietnamese,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay": 0.16, "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Chinese,Spanish,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.14, "Filipino,Chinese,Malay,English": 0.14666666666666667, "Filipino,Chinese,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian": 0.18, "Filipino,Spanish,Malay,English": 0.14666666666666667, "Filipino,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.18, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.18, "Vietnamese,Chinese,Malay,English": 0.2, "Vietnamese,Chinese,Indonesian,English": 0.2, "Vietnamese,Spanish,Malay,Indonesian": 0.18, "Vietnamese,Spanish,Malay,English": 0.18, "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Chinese,Spanish,Malay,English": 0.19333333333333333, "Chinese,Spanish,Indonesian,English": 0.18666666666666668, "Chinese,Malay,Indonesian,English": 0.19333333333333333, "Spanish,Malay,Indonesian,English": 0.18 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, "Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, "AC3_2": 0.37923362170603275, "AC3_3": 0.2852747252259087, "AC3_4": 0.2257754871514616, "AC3_5": 0.18814082044274558, "AC3_6": 0.16095852530927668, "AC3_7": 0.13788764041690293 }, "prompt_3": { "overall_acc": 0.35333333333333333, "language_acc": { "Filipino": 0.24666666666666667, "Vietnamese": 0.35333333333333333, "Chinese": 0.38, "Spanish": 0.3333333333333333, "Malay": 0.3466666666666667, "Indonesian": 0.35333333333333333, "English": 0.46 }, "consistency_score_2": 0.5107936507936508, "consistency_score_3": 0.33923809523809517, "consistency_score_4": 0.259047619047619, "consistency_score_5": 0.2146031746031746, "consistency_score_6": 0.1885714285714286, "consistency_score_7": 0.17333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.48, "Filipino,Chinese": 0.48, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.54, "Filipino,Indonesian": 0.46, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.5066666666666667, "Vietnamese,Spanish": 0.4866666666666667, "Vietnamese,Malay": 0.5666666666666667, "Vietnamese,Indonesian": 0.5, "Vietnamese,English": 0.5666666666666667, "Chinese,Spanish": 0.4666666666666667, "Chinese,Malay": 0.5266666666666666, "Chinese,Indonesian": 0.47333333333333333, "Chinese,English": 0.5333333333333333, "Spanish,Malay": 0.4666666666666667, "Spanish,Indonesian": 0.48, "Spanish,English": 0.49333333333333335, "Malay,Indonesian": 0.62, "Malay,English": 0.6, "Indonesian,English": 0.5333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.29333333333333333, "Filipino,Vietnamese,Spanish": 0.3, "Filipino,Vietnamese,Malay": 0.37333333333333335, "Filipino,Vietnamese,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,English": 0.3333333333333333, "Filipino,Chinese,Spanish": 0.29333333333333333, "Filipino,Chinese,Malay": 0.32666666666666666, "Filipino,Chinese,Indonesian": 0.2733333333333333, "Filipino,Chinese,English": 0.30666666666666664, "Filipino,Spanish,Malay": 0.32666666666666666, "Filipino,Spanish,Indonesian": 0.29333333333333333, "Filipino,Spanish,English": 0.31333333333333335, "Filipino,Malay,Indonesian": 0.36666666666666664, "Filipino,Malay,English": 0.35333333333333333, "Filipino,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish": 0.3333333333333333, "Vietnamese,Chinese,Malay": 0.36666666666666664, "Vietnamese,Chinese,Indonesian": 0.3, "Vietnamese,Chinese,English": 0.36, "Vietnamese,Spanish,Malay": 0.37333333333333335, "Vietnamese,Spanish,Indonesian": 0.32666666666666666, "Vietnamese,Spanish,English": 0.35333333333333333, "Vietnamese,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Malay,English": 0.43333333333333335, "Vietnamese,Indonesian,English": 0.37333333333333335, "Chinese,Spanish,Malay": 0.32, "Chinese,Spanish,Indonesian": 0.30666666666666664, "Chinese,Spanish,English": 0.3333333333333333, "Chinese,Malay,Indonesian": 0.36666666666666664, "Chinese,Malay,English": 0.36, "Chinese,Indonesian,English": 0.34, "Spanish,Malay,Indonesian": 0.37333333333333335, "Spanish,Malay,English": 0.3466666666666667, "Spanish,Indonesian,English": 0.31333333333333335, "Malay,Indonesian,English": 0.4266666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.28, "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Malay,English": 0.3, "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Chinese,Spanish,English": 0.24, "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Chinese,Malay,English": 0.24, "Filipino,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.26, "Filipino,Spanish,Malay,English": 0.25333333333333335, "Filipino,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,Spanish,English": 0.26, "Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, "Vietnamese,Chinese,Malay,English": 0.28, "Vietnamese,Chinese,Indonesian,English": 0.26, "Vietnamese,Spanish,Malay,Indonesian": 0.3, "Vietnamese,Spanish,Malay,English": 0.30666666666666664, "Vietnamese,Spanish,Indonesian,English": 0.26, "Vietnamese,Malay,Indonesian,English": 0.34, "Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Chinese,Spanish,Malay,English": 0.26666666666666666, "Chinese,Spanish,Indonesian,English": 0.25333333333333335, "Chinese,Malay,Indonesian,English": 0.30666666666666664, "Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,Malay,English": 0.24, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, "Filipino,Vietnamese,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.2, "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Spanish,Malay,Indonesian,English": 0.25333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.24 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 } }, "AC3_2": 0.41771736463449144, "AC3_3": 0.34614228084477683, "AC3_4": 0.29893208911655594, "AC3_5": 0.2670244083748205, "AC3_6": 0.24590509661543056, "AC3_7": 0.2325738396182877 }, "prompt_4": { "overall_acc": 0.3628571428571429, "language_acc": { "Filipino": 0.26, "Vietnamese": 0.3466666666666667, "Chinese": 0.38, "Spanish": 0.3466666666666667, "Malay": 0.4, "Indonesian": 0.38, "English": 0.4266666666666667 }, "consistency_score_2": 0.47555555555555556, "consistency_score_3": 0.2939047619047619, "consistency_score_4": 0.2106666666666667, "consistency_score_5": 0.16412698412698412, "consistency_score_6": 0.13238095238095238, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.46, "Filipino,Chinese": 0.42, "Filipino,Spanish": 0.47333333333333333, "Filipino,Malay": 0.41333333333333333, "Filipino,Indonesian": 0.4533333333333333, "Filipino,English": 0.4066666666666667, "Vietnamese,Chinese": 0.44, "Vietnamese,Spanish": 0.46, "Vietnamese,Malay": 0.5066666666666667, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.44666666666666666, "Chinese,Spanish": 0.46, "Chinese,Malay": 0.49333333333333335, "Chinese,Indonesian": 0.4266666666666667, "Chinese,English": 0.5466666666666666, "Spanish,Malay": 0.43333333333333335, "Spanish,Indonesian": 0.44666666666666666, "Spanish,English": 0.5133333333333333, "Malay,Indonesian": 0.56, "Malay,English": 0.5866666666666667, "Indonesian,English": 0.5466666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.25333333333333335, "Filipino,Vietnamese,Spanish": 0.2733333333333333, "Filipino,Vietnamese,Malay": 0.2866666666666667, "Filipino,Vietnamese,Indonesian": 0.28, "Filipino,Vietnamese,English": 0.24, "Filipino,Chinese,Spanish": 0.26, "Filipino,Chinese,Malay": 0.24666666666666667, "Filipino,Chinese,Indonesian": 0.25333333333333335, "Filipino,Chinese,English": 0.26666666666666666, "Filipino,Spanish,Malay": 0.25333333333333335, "Filipino,Spanish,Indonesian": 0.26666666666666666, "Filipino,Spanish,English": 0.28, "Filipino,Malay,Indonesian": 0.29333333333333333, "Filipino,Malay,English": 0.26666666666666666, "Filipino,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish": 0.25333333333333335, "Vietnamese,Chinese,Malay": 0.30666666666666664, "Vietnamese,Chinese,Indonesian": 0.2733333333333333, "Vietnamese,Chinese,English": 0.3, "Vietnamese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Spanish,Indonesian": 0.28, "Vietnamese,Spanish,English": 0.29333333333333333, "Vietnamese,Malay,Indonesian": 0.3466666666666667, "Vietnamese,Malay,English": 0.3333333333333333, "Vietnamese,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay": 0.2866666666666667, "Chinese,Spanish,Indonesian": 0.26, "Chinese,Spanish,English": 0.3466666666666667, "Chinese,Malay,Indonesian": 0.32666666666666666, "Chinese,Malay,English": 0.36666666666666664, "Chinese,Indonesian,English": 0.3333333333333333, "Spanish,Malay,Indonesian": 0.32, "Spanish,Malay,English": 0.32666666666666666, "Spanish,Indonesian,English": 0.32, "Malay,Indonesian,English": 0.42 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay": 0.18, "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.2, "Filipino,Chinese,Spanish,Malay": 0.18, "Filipino,Chinese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Chinese,Spanish,English": 0.21333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.2, "Filipino,Chinese,Malay,English": 0.19333333333333333, "Filipino,Chinese,Indonesian,English": 0.2, "Filipino,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Spanish,Malay,English": 0.18666666666666668, "Filipino,Spanish,Indonesian,English": 0.2, "Filipino,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.24, "Vietnamese,Chinese,Malay,English": 0.24666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.22, "Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, "Vietnamese,Malay,Indonesian,English": 0.2733333333333333, "Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Chinese,Spanish,Malay,English": 0.26, "Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Chinese,Malay,Indonesian,English": 0.28, "Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Chinese,Spanish,Malay,English": 0.16, "Filipino,Chinese,Spanish,Indonesian,English": 0.16, "Filipino,Chinese,Malay,Indonesian,English": 0.18, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.41163195754272464, "AC3_3": 0.3247613522873095, "AC3_4": 0.26656924605116283, "AC3_5": 0.22602065400186097, "AC3_6": 0.19398901094984014, "AC3_7": 0.1648681541231011 }, "prompt_5": { "overall_acc": 0.3380952380952381, "language_acc": { "Filipino": 0.23333333333333334, "Vietnamese": 0.3466666666666667, "Chinese": 0.35333333333333333, "Spanish": 0.32666666666666666, "Malay": 0.32666666666666666, "Indonesian": 0.37333333333333335, "English": 0.4066666666666667 }, "consistency_score_2": 0.5336507936507936, "consistency_score_3": 0.3683809523809523, "consistency_score_4": 0.2862857142857142, "consistency_score_5": 0.23682539682539688, "consistency_score_6": 0.20380952380952383, "consistency_score_7": 0.18, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.49333333333333335, "Filipino,Chinese": 0.4866666666666667, "Filipino,Spanish": 0.43333333333333335, "Filipino,Malay": 0.4866666666666667, "Filipino,Indonesian": 0.5066666666666667, "Filipino,English": 0.47333333333333333, "Vietnamese,Chinese": 0.5133333333333333, "Vietnamese,Spanish": 0.5333333333333333, "Vietnamese,Malay": 0.5866666666666667, "Vietnamese,Indonesian": 0.5733333333333334, "Vietnamese,English": 0.5933333333333334, "Chinese,Spanish": 0.52, "Chinese,Malay": 0.5733333333333334, "Chinese,Indonesian": 0.5333333333333333, "Chinese,English": 0.6066666666666667, "Spanish,Malay": 0.5066666666666667, "Spanish,Indonesian": 0.4866666666666667, "Spanish,English": 0.5666666666666667, "Malay,Indonesian": 0.6266666666666667, "Malay,English": 0.5333333333333333, "Indonesian,English": 0.5733333333333334 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.31333333333333335, "Filipino,Vietnamese,Spanish": 0.31333333333333335, "Filipino,Vietnamese,Malay": 0.36666666666666664, "Filipino,Vietnamese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,English": 0.3466666666666667, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.32666666666666666, "Filipino,Chinese,Indonesian": 0.32666666666666666, "Filipino,Chinese,English": 0.34, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.26666666666666666, "Filipino,Spanish,English": 0.3, "Filipino,Malay,Indonesian": 0.38, "Filipino,Malay,English": 0.31333333333333335, "Filipino,Indonesian,English": 0.32, "Vietnamese,Chinese,Spanish": 0.36666666666666664, "Vietnamese,Chinese,Malay": 0.41333333333333333, "Vietnamese,Chinese,Indonesian": 0.38, "Vietnamese,Chinese,English": 0.41333333333333333, "Vietnamese,Spanish,Malay": 0.3933333333333333, "Vietnamese,Spanish,Indonesian": 0.36666666666666664, "Vietnamese,Spanish,English": 0.41333333333333333, "Vietnamese,Malay,Indonesian": 0.46, "Vietnamese,Malay,English": 0.44, "Vietnamese,Indonesian,English": 0.44, "Chinese,Spanish,Malay": 0.38, "Chinese,Spanish,Indonesian": 0.35333333333333333, "Chinese,Spanish,English": 0.4066666666666667, "Chinese,Malay,Indonesian": 0.4266666666666667, "Chinese,Malay,English": 0.4066666666666667, "Chinese,Indonesian,English": 0.4066666666666667, "Spanish,Malay,Indonesian": 0.3933333333333333, "Spanish,Malay,English": 0.38, "Spanish,Indonesian,English": 0.37333333333333335, "Malay,Indonesian,English": 0.42 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.28, "Filipino,Vietnamese,Chinese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.32, "Filipino,Vietnamese,Malay,English": 0.29333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.28, "Filipino,Chinese,Spanish,Malay": 0.24666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Chinese,Spanish,English": 0.24, "Filipino,Chinese,Malay,Indonesian": 0.2733333333333333, "Filipino,Chinese,Malay,English": 0.26, "Filipino,Chinese,Indonesian,English": 0.25333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Spanish,Malay,English": 0.24, "Filipino,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,Spanish,English": 0.31333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.3466666666666667, "Vietnamese,Chinese,Malay,English": 0.3466666666666667, "Vietnamese,Chinese,Indonesian,English": 0.32666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.3333333333333333, "Vietnamese,Spanish,Malay,English": 0.3333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, "Vietnamese,Malay,Indonesian,English": 0.37333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, "Chinese,Spanish,Malay,English": 0.3333333333333333, "Chinese,Spanish,Indonesian,English": 0.31333333333333335, "Chinese,Malay,Indonesian,English": 0.3333333333333333, "Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.2, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.24, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.22, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.26, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.22, "Filipino,Chinese,Spanish,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, "Vietnamese,Chinese,Spanish,Malay,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, "Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.2, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 } }, "AC3_2": 0.41393888941944634, "AC3_3": 0.35258894062207374, "AC3_4": 0.310040965696008, "AC3_5": 0.2785411900333268, "AC3_6": 0.2543141685027398, "AC3_7": 0.234926470542891 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3262987012987013, "language_acc": { "Indonesian": 0.3352272727272727, "English": 0.32386363636363635, "Filipino": 0.2897727272727273, "Spanish": 0.32954545454545453, "Chinese": 0.35795454545454547, "Malay": 0.3352272727272727, "Vietnamese": 0.3125 }, "consistency_score_2": 0.4897186147186148, "consistency_score_3": 0.30584415584415586, "consistency_score_4": 0.2103896103896104, "consistency_score_5": 0.15313852813852813, "consistency_score_6": 0.11607142857142858, "consistency_score_7": 0.09090909090909091, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5852272727272727, "Indonesian,Filipino": 0.3465909090909091, "Indonesian,Spanish": 0.5227272727272727, "Indonesian,Chinese": 0.48295454545454547, "Indonesian,Malay": 0.6022727272727273, "Indonesian,Vietnamese": 0.5568181818181818, "English,Filipino": 0.3352272727272727, "English,Spanish": 0.5170454545454546, "English,Chinese": 0.5056818181818182, "English,Malay": 0.5454545454545454, "English,Vietnamese": 0.5965909090909091, "Filipino,Spanish": 0.4772727272727273, "Filipino,Chinese": 0.2840909090909091, "Filipino,Malay": 0.4034090909090909, "Filipino,Vietnamese": 0.3522727272727273, "Spanish,Chinese": 0.4375, "Spanish,Malay": 0.6420454545454546, "Spanish,Vietnamese": 0.5511363636363636, "Chinese,Malay": 0.4659090909090909, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.2215909090909091, "Indonesian,English,Spanish": 0.36363636363636365, "Indonesian,English,Chinese": 0.3465909090909091, "Indonesian,English,Malay": 0.4147727272727273, "Indonesian,English,Vietnamese": 0.42613636363636365, "Indonesian,Filipino,Spanish": 0.26136363636363635, "Indonesian,Filipino,Chinese": 0.16477272727272727, "Indonesian,Filipino,Malay": 0.24431818181818182, "Indonesian,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Chinese": 0.29545454545454547, "Indonesian,Spanish,Malay": 0.4431818181818182, "Indonesian,Spanish,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay": 0.3522727272727273, "Indonesian,Chinese,Vietnamese": 0.3465909090909091, "Indonesian,Malay,Vietnamese": 0.4147727272727273, "English,Filipino,Spanish": 0.23295454545454544, "English,Filipino,Chinese": 0.1875, "English,Filipino,Malay": 0.20454545454545456, "English,Filipino,Vietnamese": 0.21022727272727273, "English,Spanish,Chinese": 0.2897727272727273, "English,Spanish,Malay": 0.39204545454545453, "English,Spanish,Vietnamese": 0.38636363636363635, "English,Chinese,Malay": 0.3181818181818182, "English,Chinese,Vietnamese": 0.3522727272727273, "English,Malay,Vietnamese": 0.4090909090909091, "Filipino,Spanish,Chinese": 0.19886363636363635, "Filipino,Spanish,Malay": 0.32954545454545453, "Filipino,Spanish,Vietnamese": 0.25, "Filipino,Chinese,Malay": 0.20454545454545456, "Filipino,Chinese,Vietnamese": 0.18181818181818182, "Filipino,Malay,Vietnamese": 0.24431818181818182, "Spanish,Chinese,Malay": 0.32954545454545453, "Spanish,Chinese,Vietnamese": 0.3181818181818182, "Spanish,Malay,Vietnamese": 0.42613636363636365, "Chinese,Malay,Vietnamese": 0.3522727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.17045454545454544, "Indonesian,English,Filipino,Chinese": 0.14204545454545456, "Indonesian,English,Filipino,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese": 0.23295454545454544, "Indonesian,English,Spanish,Malay": 0.3181818181818182, "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Spanish,Chinese,Malay": 0.2556818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.25, "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.2840909090909091, "English,Filipino,Spanish,Chinese": 0.13636363636363635, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, "English,Filipino,Chinese,Malay": 0.14772727272727273, "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, "English,Filipino,Malay,Vietnamese": 0.1534090909090909, "English,Spanish,Chinese,Malay": 0.23863636363636365, "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, "English,Spanish,Malay,Vietnamese": 0.3125, "English,Chinese,Malay,Vietnamese": 0.2727272727272727, "Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.11931818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.125, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.125, "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 } }, "AC3_2": 0.39164499289010174, "AC3_3": 0.31574049981998215, "AC3_4": 0.25582765686891834, "AC3_5": 0.20844815451077167, "AC3_6": 0.17123197899143633, "AC3_7": 0.14220021220503895 }, "prompt_2": { "overall_acc": 0.3092532467532468, "language_acc": { "Indonesian": 0.2840909090909091, "English": 0.38636363636363635, "Filipino": 0.26704545454545453, "Spanish": 0.29545454545454547, "Chinese": 0.3068181818181818, "Malay": 0.3068181818181818, "Vietnamese": 0.3181818181818182 }, "consistency_score_2": 0.45670995670995673, "consistency_score_3": 0.26120129870129866, "consistency_score_4": 0.1642857142857143, "consistency_score_5": 0.10876623376623377, "consistency_score_6": 0.07386363636363637, "consistency_score_7": 0.05113636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5454545454545454, "Indonesian,Filipino": 0.2784090909090909, "Indonesian,Spanish": 0.48295454545454547, "Indonesian,Chinese": 0.4602272727272727, "Indonesian,Malay": 0.5738636363636364, "Indonesian,Vietnamese": 0.5681818181818182, "English,Filipino": 0.30113636363636365, "English,Spanish": 0.48295454545454547, "English,Chinese": 0.4943181818181818, "English,Malay": 0.5056818181818182, "English,Vietnamese": 0.5852272727272727, "Filipino,Spanish": 0.4375, "Filipino,Chinese": 0.3352272727272727, "Filipino,Malay": 0.30113636363636365, "Filipino,Vietnamese": 0.3181818181818182, "Spanish,Chinese": 0.42613636363636365, "Spanish,Malay": 0.5056818181818182, "Spanish,Vietnamese": 0.5227272727272727, "Chinese,Malay": 0.44886363636363635, "Chinese,Vietnamese": 0.5, "Malay,Vietnamese": 0.5170454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.1590909090909091, "Indonesian,English,Spanish": 0.32954545454545453, "Indonesian,English,Chinese": 0.3125, "Indonesian,English,Malay": 0.375, "Indonesian,English,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish": 0.19886363636363635, "Indonesian,Filipino,Chinese": 0.1534090909090909, "Indonesian,Filipino,Malay": 0.17045454545454544, "Indonesian,Filipino,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese": 0.2556818181818182, "Indonesian,Spanish,Malay": 0.3409090909090909, "Indonesian,Spanish,Vietnamese": 0.36363636363636365, "Indonesian,Chinese,Malay": 0.3068181818181818, "Indonesian,Chinese,Vietnamese": 0.3181818181818182, "Indonesian,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish": 0.1875, "English,Filipino,Chinese": 0.17613636363636365, "English,Filipino,Malay": 0.1590909090909091, "English,Filipino,Vietnamese": 0.1875, "English,Spanish,Chinese": 0.26136363636363635, "English,Spanish,Malay": 0.30113636363636365, "English,Spanish,Vietnamese": 0.36363636363636365, "English,Chinese,Malay": 0.29545454545454547, "English,Chinese,Vietnamese": 0.3522727272727273, "English,Malay,Vietnamese": 0.3409090909090909, "Filipino,Spanish,Chinese": 0.21022727272727273, "Filipino,Spanish,Malay": 0.2159090909090909, "Filipino,Spanish,Vietnamese": 0.2159090909090909, "Filipino,Chinese,Malay": 0.1534090909090909, "Filipino,Chinese,Vietnamese": 0.16477272727272727, "Filipino,Malay,Vietnamese": 0.1590909090909091, "Spanish,Chinese,Malay": 0.26704545454545453, "Spanish,Chinese,Vietnamese": 0.2840909090909091, "Spanish,Malay,Vietnamese": 0.32954545454545453, "Chinese,Malay,Vietnamese": 0.29545454545454547 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.125, "Indonesian,English,Filipino,Chinese": 0.10795454545454546, "Indonesian,English,Filipino,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Spanish,Malay": 0.23863636363636365, "Indonesian,English,Spanish,Vietnamese": 0.26136363636363635, "Indonesian,English,Chinese,Malay": 0.22727272727272727, "Indonesian,English,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,English,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Malay,Vietnamese": 0.125, "Indonesian,Spanish,Chinese,Malay": 0.19886363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.23295454545454544, "English,Filipino,Spanish,Chinese": 0.11931818181818182, "English,Filipino,Spanish,Malay": 0.125, "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, "English,Filipino,Chinese,Malay": 0.10795454545454546, "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, "English,Filipino,Malay,Vietnamese": 0.10227272727272728, "English,Spanish,Chinese,Malay": 0.1875, "English,Spanish,Chinese,Vietnamese": 0.22727272727272727, "English,Spanish,Malay,Vietnamese": 0.2215909090909091, "English,Chinese,Malay,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Filipino,Spanish,Chinese,Vietnamese": 0.125, "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0625, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 } }, "AC3_2": 0.36878804694548106, "AC3_3": 0.28320345699416466, "AC3_4": 0.21457955820925284, "AC3_5": 0.16093178662144336, "AC3_6": 0.11924595528474537, "AC3_7": 0.08776105648670479 }, "prompt_3": { "overall_acc": 0.32711038961038963, "language_acc": { "Indonesian": 0.30113636363636365, "English": 0.39204545454545453, "Filipino": 0.3068181818181818, "Spanish": 0.3068181818181818, "Chinese": 0.375, "Malay": 0.3181818181818182, "Vietnamese": 0.2897727272727273 }, "consistency_score_2": 0.49323593073593064, "consistency_score_3": 0.3066558441558441, "consistency_score_4": 0.20925324675324675, "consistency_score_5": 0.14989177489177488, "consistency_score_6": 0.10957792207792207, "consistency_score_7": 0.07954545454545454, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5681818181818182, "Indonesian,Filipino": 0.32386363636363635, "Indonesian,Spanish": 0.5227272727272727, "Indonesian,Chinese": 0.44886363636363635, "Indonesian,Malay": 0.6022727272727273, "Indonesian,Vietnamese": 0.5909090909090909, "English,Filipino": 0.32386363636363635, "English,Spanish": 0.5568181818181818, "English,Chinese": 0.5340909090909091, "English,Malay": 0.5625, "English,Vietnamese": 0.5568181818181818, "Filipino,Spanish": 0.4715909090909091, "Filipino,Chinese": 0.3522727272727273, "Filipino,Malay": 0.42613636363636365, "Filipino,Vietnamese": 0.32954545454545453, "Spanish,Chinese": 0.5056818181818182, "Spanish,Malay": 0.5795454545454546, "Spanish,Vietnamese": 0.5625, "Chinese,Malay": 0.4943181818181818, "Chinese,Vietnamese": 0.5, "Malay,Vietnamese": 0.5454545454545454 }, "3_combine": { "Indonesian,English,Filipino": 0.20454545454545456, "Indonesian,English,Spanish": 0.36363636363636365, "Indonesian,English,Chinese": 0.32954545454545453, "Indonesian,English,Malay": 0.4147727272727273, "Indonesian,English,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Spanish": 0.24431818181818182, "Indonesian,Filipino,Chinese": 0.16477272727272727, "Indonesian,Filipino,Malay": 0.25, "Indonesian,Filipino,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese": 0.30113636363636365, "Indonesian,Spanish,Malay": 0.38636363636363635, "Indonesian,Spanish,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay": 0.3352272727272727, "Indonesian,Chinese,Vietnamese": 0.3409090909090909, "Indonesian,Malay,Vietnamese": 0.42045454545454547, "English,Filipino,Spanish": 0.23863636363636365, "English,Filipino,Chinese": 0.19318181818181818, "English,Filipino,Malay": 0.23295454545454544, "English,Filipino,Vietnamese": 0.19318181818181818, "English,Spanish,Chinese": 0.3522727272727273, "English,Spanish,Malay": 0.3977272727272727, "English,Spanish,Vietnamese": 0.3977272727272727, "English,Chinese,Malay": 0.35795454545454547, "English,Chinese,Vietnamese": 0.3522727272727273, "English,Malay,Vietnamese": 0.3977272727272727, "Filipino,Spanish,Chinese": 0.23863636363636365, "Filipino,Spanish,Malay": 0.3181818181818182, "Filipino,Spanish,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay": 0.2215909090909091, "Filipino,Chinese,Vietnamese": 0.19318181818181818, "Filipino,Malay,Vietnamese": 0.23295454545454544, "Spanish,Chinese,Malay": 0.3522727272727273, "Spanish,Chinese,Vietnamese": 0.3465909090909091, "Spanish,Malay,Vietnamese": 0.3977272727272727, "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1590909090909091, "Indonesian,English,Filipino,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Vietnamese": 0.14204545454545456, "Indonesian,English,Spanish,Chinese": 0.24431818181818182, "Indonesian,English,Spanish,Malay": 0.2840909090909091, "Indonesian,English,Spanish,Vietnamese": 0.29545454545454547, "Indonesian,English,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.125, "Indonesian,Filipino,Spanish,Malay": 0.21022727272727273, "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Chinese,Malay": 0.13068181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.125, "Indonesian,Filipino,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Spanish,Chinese,Malay": 0.24431818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.3125, "Indonesian,Chinese,Malay,Vietnamese": 0.2727272727272727, "English,Filipino,Spanish,Chinese": 0.1534090909090909, "English,Filipino,Spanish,Malay": 0.19886363636363635, "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, "English,Filipino,Chinese,Malay": 0.1534090909090909, "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, "English,Filipino,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay": 0.2727272727272727, "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, "English,Spanish,Malay,Vietnamese": 0.32386363636363635, "English,Chinese,Malay,Vietnamese": 0.26704545454545453, "Filipino,Spanish,Chinese,Malay": 0.1875, "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.125, "Indonesian,English,Filipino,Chinese,Malay": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.19886363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 } }, "AC3_2": 0.393352401178792, "AC3_3": 0.31655303579740146, "AC3_4": 0.25523322768110196, "AC3_5": 0.2055804376700676, "AC3_6": 0.16416320664420125, "AC3_7": 0.12797133003566927 }, "prompt_4": { "overall_acc": 0.31818181818181823, "language_acc": { "Indonesian": 0.32386363636363635, "English": 0.32954545454545453, "Filipino": 0.2897727272727273, "Spanish": 0.3068181818181818, "Chinese": 0.36363636363636365, "Malay": 0.3465909090909091, "Vietnamese": 0.26704545454545453 }, "consistency_score_2": 0.47781385281385286, "consistency_score_3": 0.2883116883116883, "consistency_score_4": 0.18863636363636363, "consistency_score_5": 0.12662337662337664, "consistency_score_6": 0.08360389610389611, "consistency_score_7": 0.05113636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5454545454545454, "Indonesian,Filipino": 0.3125, "Indonesian,Spanish": 0.5227272727272727, "Indonesian,Chinese": 0.45454545454545453, "Indonesian,Malay": 0.5454545454545454, "Indonesian,Vietnamese": 0.5340909090909091, "English,Filipino": 0.2840909090909091, "English,Spanish": 0.5397727272727273, "English,Chinese": 0.5113636363636364, "English,Malay": 0.5738636363636364, "English,Vietnamese": 0.5397727272727273, "Filipino,Spanish": 0.48863636363636365, "Filipino,Chinese": 0.3068181818181818, "Filipino,Malay": 0.4034090909090909, "Filipino,Vietnamese": 0.3522727272727273, "Spanish,Chinese": 0.44886363636363635, "Spanish,Malay": 0.5511363636363636, "Spanish,Vietnamese": 0.5511363636363636, "Chinese,Malay": 0.5, "Chinese,Vietnamese": 0.48295454545454547, "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.16477272727272727, "Indonesian,English,Spanish": 0.36363636363636365, "Indonesian,English,Chinese": 0.32386363636363635, "Indonesian,English,Malay": 0.3806818181818182, "Indonesian,English,Vietnamese": 0.36363636363636365, "Indonesian,Filipino,Spanish": 0.22727272727272727, "Indonesian,Filipino,Chinese": 0.1534090909090909, "Indonesian,Filipino,Malay": 0.20454545454545456, "Indonesian,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese": 0.29545454545454547, "Indonesian,Spanish,Malay": 0.3693181818181818, "Indonesian,Spanish,Vietnamese": 0.36363636363636365, "Indonesian,Chinese,Malay": 0.3125, "Indonesian,Chinese,Vietnamese": 0.3125, "Indonesian,Malay,Vietnamese": 0.4034090909090909, "English,Filipino,Spanish": 0.22727272727272727, "English,Filipino,Chinese": 0.1534090909090909, "English,Filipino,Malay": 0.20454545454545456, "English,Filipino,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese": 0.30113636363636365, "English,Spanish,Malay": 0.375, "English,Spanish,Vietnamese": 0.375, "English,Chinese,Malay": 0.3465909090909091, "English,Chinese,Vietnamese": 0.3352272727272727, "English,Malay,Vietnamese": 0.3977272727272727, "Filipino,Spanish,Chinese": 0.22727272727272727, "Filipino,Spanish,Malay": 0.2897727272727273, "Filipino,Spanish,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay": 0.20454545454545456, "Filipino,Chinese,Vietnamese": 0.17613636363636365, "Filipino,Malay,Vietnamese": 0.24431818181818182, "Spanish,Chinese,Malay": 0.3125, "Spanish,Chinese,Vietnamese": 0.32386363636363635, "Spanish,Malay,Vietnamese": 0.38636363636363635, "Chinese,Malay,Vietnamese": 0.36363636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.13636363636363635, "Indonesian,English,Filipino,Chinese": 0.09090909090909091, "Indonesian,English,Filipino,Malay": 0.11931818181818182, "Indonesian,English,Filipino,Vietnamese": 0.11931818181818182, "Indonesian,English,Spanish,Chinese": 0.22727272727272727, "Indonesian,English,Spanish,Malay": 0.2840909090909091, "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,English,Chinese,Malay": 0.24431818181818182, "Indonesian,English,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Malay,Vietnamese": 0.29545454545454547, "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay": 0.22727272727272727, "Indonesian,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.29545454545454547, "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese": 0.13068181818181818, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.1590909090909091, "English,Filipino,Chinese,Malay": 0.11931818181818182, "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, "English,Filipino,Malay,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese,Malay": 0.22727272727272727, "English,Spanish,Chinese,Vietnamese": 0.23863636363636365, "English,Spanish,Malay,Vietnamese": 0.2840909090909091, "English,Chinese,Malay,Vietnamese": 0.26704545454545453, "Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.07954545454545454, "Indonesian,English,Filipino,Spanish,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, "English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.125, "English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0625, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 } }, "AC3_2": 0.3819912242273957, "AC3_3": 0.3025111932536058, "AC3_4": 0.23685283321865602, "AC3_5": 0.1811546117708194, "AC3_6": 0.13241505965483033, "AC3_7": 0.08811188808803014 }, "prompt_5": { "overall_acc": 0.32224025974025977, "language_acc": { "Indonesian": 0.3409090909090909, "English": 0.32954545454545453, "Filipino": 0.2897727272727273, "Spanish": 0.3125, "Chinese": 0.3125, "Malay": 0.35795454545454547, "Vietnamese": 0.3125 }, "consistency_score_2": 0.4783549783549783, "consistency_score_3": 0.287012987012987, "consistency_score_4": 0.1910714285714286, "consistency_score_5": 0.13419913419913423, "consistency_score_6": 0.09659090909090909, "consistency_score_7": 0.06818181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5738636363636364, "Indonesian,Filipino": 0.32954545454545453, "Indonesian,Spanish": 0.5113636363636364, "Indonesian,Chinese": 0.4318181818181818, "Indonesian,Malay": 0.5340909090909091, "Indonesian,Vietnamese": 0.5681818181818182, "English,Filipino": 0.3352272727272727, "English,Spanish": 0.5625, "English,Chinese": 0.5397727272727273, "English,Malay": 0.5284090909090909, "English,Vietnamese": 0.5170454545454546, "Filipino,Spanish": 0.4659090909090909, "Filipino,Chinese": 0.3125, "Filipino,Malay": 0.375, "Filipino,Vietnamese": 0.3977272727272727, "Spanish,Chinese": 0.4943181818181818, "Spanish,Malay": 0.5681818181818182, "Spanish,Vietnamese": 0.5625, "Chinese,Malay": 0.44886363636363635, "Chinese,Vietnamese": 0.45454545454545453, "Malay,Vietnamese": 0.5340909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.21022727272727273, "Indonesian,English,Spanish": 0.375, "Indonesian,English,Chinese": 0.3181818181818182, "Indonesian,English,Malay": 0.375, "Indonesian,English,Vietnamese": 0.3806818181818182, "Indonesian,Filipino,Spanish": 0.23295454545454544, "Indonesian,Filipino,Chinese": 0.14204545454545456, "Indonesian,Filipino,Malay": 0.19886363636363635, "Indonesian,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Chinese": 0.2897727272727273, "Indonesian,Spanish,Malay": 0.35795454545454547, "Indonesian,Spanish,Vietnamese": 0.35795454545454547, "Indonesian,Chinese,Malay": 0.2784090909090909, "Indonesian,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Malay,Vietnamese": 0.375, "English,Filipino,Spanish": 0.26136363636363635, "English,Filipino,Chinese": 0.18181818181818182, "English,Filipino,Malay": 0.2215909090909091, "English,Filipino,Vietnamese": 0.2215909090909091, "English,Spanish,Chinese": 0.3409090909090909, "English,Spanish,Malay": 0.39204545454545453, "English,Spanish,Vietnamese": 0.36363636363636365, "English,Chinese,Malay": 0.32386363636363635, "English,Chinese,Vietnamese": 0.32954545454545453, "English,Malay,Vietnamese": 0.3409090909090909, "Filipino,Spanish,Chinese": 0.23295454545454544, "Filipino,Spanish,Malay": 0.2784090909090909, "Filipino,Spanish,Vietnamese": 0.2727272727272727, "Filipino,Chinese,Malay": 0.14772727272727273, "Filipino,Chinese,Vietnamese": 0.1875, "Filipino,Malay,Vietnamese": 0.23863636363636365, "Spanish,Chinese,Malay": 0.3068181818181818, "Spanish,Chinese,Vietnamese": 0.32386363636363635, "Spanish,Malay,Vietnamese": 0.375, "Chinese,Malay,Vietnamese": 0.3068181818181818 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.17613636363636365, "Indonesian,English,Filipino,Chinese": 0.125, "Indonesian,English,Filipino,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese": 0.23295454545454544, "Indonesian,English,Spanish,Malay": 0.2840909090909091, "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,English,Chinese,Malay": 0.2215909090909091, "Indonesian,English,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,English,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese": 0.125, "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay": 0.09090909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Spanish,Chinese,Malay": 0.20454545454545456, "Indonesian,Spanish,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish,Chinese": 0.1590909090909091, "English,Filipino,Spanish,Malay": 0.19886363636363635, "English,Filipino,Spanish,Vietnamese": 0.1875, "English,Filipino,Chinese,Malay": 0.11363636363636363, "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, "English,Filipino,Malay,Vietnamese": 0.16477272727272727, "English,Spanish,Chinese,Malay": 0.25, "English,Spanish,Chinese,Vietnamese": 0.2556818181818182, "English,Spanish,Malay,Vietnamese": 0.26704545454545453, "English,Chinese,Malay,Vietnamese": 0.22727272727272727, "Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Malay,Vietnamese": 0.125, "Indonesian,English,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,English,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,English,Chinese,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, "English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 } }, "AC3_2": 0.38507656583279287, "AC3_3": 0.3036081956446938, "AC3_4": 0.23989676511094088, "AC3_5": 0.1894856772924148, "AC3_6": 0.14863019728657087, "AC3_7": 0.11254961252078487 } }, "sg_eval": { "prompt_1": { "accuracy": 0.3592233009708738 }, "prompt_2": { "accuracy": 0.3883495145631068 }, "prompt_3": { "accuracy": 0.39805825242718446 }, "prompt_4": { "accuracy": 0.3786407766990291 }, "prompt_5": { "accuracy": 0.34951456310679613 } }, "cn_eval": { "prompt_1": { "accuracy": 0.29523809523809524 }, "prompt_2": { "accuracy": 0.3523809523809524 }, "prompt_3": { "accuracy": 0.3142857142857143 }, "prompt_4": { "accuracy": 0.26666666666666666 }, "prompt_5": { "accuracy": 0.3142857142857143 } }, "us_eval": { "prompt_1": { "accuracy": 0.411214953271028 }, "prompt_2": { "accuracy": 0.34579439252336447 }, "prompt_3": { "accuracy": 0.411214953271028 }, "prompt_4": { "accuracy": 0.38317757009345793 }, "prompt_5": { "accuracy": 0.3925233644859813 } }, "ph_eval": { "prompt_1": { "accuracy": 0.27, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.3, "history": 0.3333333333333333, "literature": 0.4, "politics": 0.1, "culture": 0.2, "film": 0.3, "law": 0.1, "geography": 0.5 } }, "prompt_2": { "accuracy": 0.29, "category_acc": { "brand": 0.1, "demographics": 0.2, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.5, "politics": 0.0, "culture": 0.2, "film": 0.3, "law": 0.3, "geography": 0.5 } }, "prompt_3": { "accuracy": 0.27, "category_acc": { "brand": 0.3, "demographics": 0.0, "biology": 0.6, "history": 0.2, "literature": 0.3, "politics": 0.1, "culture": 0.2, "film": 0.2, "law": 0.2, "geography": 0.5 } }, "prompt_4": { "accuracy": 0.28, "category_acc": { "brand": 0.3, "demographics": 0.0, "biology": 0.4, "history": 0.2, "literature": 0.4, "politics": 0.3, "culture": 0.3, "film": 0.2, "law": 0.2, "geography": 0.4 } }, "prompt_5": { "accuracy": 0.24, "category_acc": { "brand": 0.2, "demographics": 0.0, "biology": 0.6, "history": 0.13333333333333333, "literature": 0.4, "politics": 0.2, "culture": 0.3, "film": 0.1, "law": 0.0, "geography": 0.4 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.06305082715215274 }, "prompt_2": { "bleu_score": 0.06463430550758704 }, "prompt_3": { "bleu_score": 0.06356003237143983 }, "prompt_4": { "bleu_score": 0.06539930841182513 }, "prompt_5": { "bleu_score": 0.06031168781615755 } }, "indommlu": { "prompt_1": { "accuracy": 0.3378730222311236, "category_acc": { "History": 0.3092369477911647, "Geography": 0.25918367346938775, "Lampungic": 0.272108843537415, "Social science": 0.4774624373956594, "Balinese": 0.28450106157112526, "Makassarese": 0.2956989247311828, "Banjarese": 0.3541666666666667, "Chemistry": 0.20583941605839415, "Biology": 0.27455621301775146, "Science": 0.3849329205366357, "Christian religion": 0.38308457711442784, "Art": 0.36605657237936773, "Islam religion": 0.3968705547652916, "Hindu religion": 0.36666666666666664, "Madurese": 0.2847457627118644, "Sport": 0.34459459459459457, "Indonesian language": 0.37391033623910336, "Physics": 0.298989898989899, "Minangkabau culture": 0.3417085427135678, "Dayak language": 0.3486238532110092, "Sociology": 0.29435483870967744, "Economy": 0.305327868852459, "Sundanese": 0.32757130509939497, "Javanese": 0.3084677419354839, "Civic education": 0.38197424892703863 } }, "prompt_2": { "accuracy": 0.3440817144001602, "category_acc": { "History": 0.3172690763052209, "Geography": 0.2836734693877551, "Lampungic": 0.2585034013605442, "Social science": 0.49248747913188645, "Balinese": 0.29936305732484075, "Makassarese": 0.3333333333333333, "Banjarese": 0.3125, "Chemistry": 0.21897810218978103, "Biology": 0.25088757396449707, "Science": 0.39628482972136225, "Christian religion": 0.39800995024875624, "Art": 0.37603993344425957, "Islam religion": 0.4167852062588905, "Hindu religion": 0.35333333333333333, "Madurese": 0.31864406779661014, "Sport": 0.39864864864864863, "Indonesian language": 0.3866749688667497, "Physics": 0.27070707070707073, "Minangkabau culture": 0.33668341708542715, "Dayak language": 0.3211009174311927, "Sociology": 0.2923387096774194, "Economy": 0.2581967213114754, "Sundanese": 0.3327571305099395, "Javanese": 0.3215725806451613, "Civic education": 0.38912732474964234 } }, "prompt_3": { "accuracy": 0.3464183189799052, "category_acc": { "History": 0.3032128514056225, "Geography": 0.2510204081632653, "Lampungic": 0.2653061224489796, "Social science": 0.49081803005008345, "Balinese": 0.2823779193205945, "Makassarese": 0.3333333333333333, "Banjarese": 0.3263888888888889, "Chemistry": 0.22043795620437956, "Biology": 0.2603550295857988, "Science": 0.40970072239422084, "Christian religion": 0.40298507462686567, "Art": 0.39267886855241263, "Islam religion": 0.41109530583214793, "Hindu religion": 0.35333333333333333, "Madurese": 0.3016949152542373, "Sport": 0.3918918918918919, "Indonesian language": 0.3866749688667497, "Physics": 0.30303030303030304, "Minangkabau culture": 0.3316582914572864, "Dayak language": 0.3302752293577982, "Sociology": 0.29838709677419356, "Economy": 0.29918032786885246, "Sundanese": 0.34485738980121, "Javanese": 0.3064516129032258, "Civic education": 0.39341917024320455 } }, "prompt_4": { "accuracy": 0.3415448294278657, "category_acc": { "History": 0.3232931726907631, "Geography": 0.2673469387755102, "Lampungic": 0.32653061224489793, "Social science": 0.48414023372287146, "Balinese": 0.2760084925690021, "Makassarese": 0.24731182795698925, "Banjarese": 0.3125, "Chemistry": 0.24817518248175183, "Biology": 0.2733727810650888, "Science": 0.391124871001032, "Christian religion": 0.39800995024875624, "Art": 0.37104825291181365, "Islam religion": 0.39544807965860596, "Hindu religion": 0.35333333333333333, "Madurese": 0.2711864406779661, "Sport": 0.3581081081081081, "Indonesian language": 0.3941469489414695, "Physics": 0.2606060606060606, "Minangkabau culture": 0.2914572864321608, "Dayak language": 0.22935779816513763, "Sociology": 0.2862903225806452, "Economy": 0.29713114754098363, "Sundanese": 0.31892826274848746, "Javanese": 0.30443548387096775, "Civic education": 0.4034334763948498 } }, "prompt_5": { "accuracy": 0.34681887976500436, "category_acc": { "History": 0.3473895582329317, "Geography": 0.2816326530612245, "Lampungic": 0.35374149659863946, "Social science": 0.5008347245409015, "Balinese": 0.2781316348195329, "Makassarese": 0.27956989247311825, "Banjarese": 0.3472222222222222, "Chemistry": 0.24817518248175183, "Biology": 0.2591715976331361, "Science": 0.4004127966976264, "Christian religion": 0.3781094527363184, "Art": 0.3826955074875208, "Islam religion": 0.40825035561877665, "Hindu religion": 0.38, "Madurese": 0.28135593220338984, "Sport": 0.40540540540540543, "Indonesian language": 0.37951432129514323, "Physics": 0.298989898989899, "Minangkabau culture": 0.36180904522613067, "Dayak language": 0.26605504587155965, "Sociology": 0.2903225806451613, "Economy": 0.2766393442622951, "Sundanese": 0.34312878133102853, "Javanese": 0.3155241935483871, "Civic education": 0.38912732474964234 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.07908898359419463 }, "prompt_2": { "bleu_score": 0.07897628396673814 }, "prompt_3": { "bleu_score": 0.07899232385079807 }, "prompt_4": { "bleu_score": 0.07891920764416933 }, "prompt_5": { "bleu_score": 0.07468075077628368 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.06631314380326508 }, "prompt_2": { "bleu_score": 0.06715912321244885 }, "prompt_3": { "bleu_score": 0.0669243196705807 }, "prompt_4": { "bleu_score": 0.06705363387875442 }, "prompt_5": { "bleu_score": 0.06463296313543722 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.04429631011508256 }, "prompt_2": { "bleu_score": 0.04422337129052579 }, "prompt_3": { "bleu_score": 0.044872517066552334 }, "prompt_4": { "bleu_score": 0.04557580925559713 }, "prompt_5": { "bleu_score": 0.04355200215183761 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.07958207120800556 }, "prompt_2": { "bleu_score": 0.07948625371017644 }, "prompt_3": { "bleu_score": 0.07995669494424623 }, "prompt_4": { "bleu_score": 0.07923605117741923 }, "prompt_5": { "bleu_score": 0.07715463343756887 } }, "mmlu": { "prompt_1": { "accuracy": 0.35939323220536756 }, "prompt_2": { "accuracy": 0.35122520420070014 }, "prompt_3": { "accuracy": 0.35822637106184363 }, "prompt_4": { "accuracy": 0.34072345390898484 }, "prompt_5": { "accuracy": 0.367561260210035 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.36467643904183056, "category_acc": { "high_school_european_history": 0.45121951219512196, "business_ethics": 0.3838383838383838, "clinical_knowledge": 0.4128787878787879, "medical_genetics": 0.3939393939393939, "high_school_us_history": 0.4482758620689655, "high_school_physics": 0.26666666666666666, "high_school_world_history": 0.4957627118644068, "virology": 0.3151515151515151, "high_school_microeconomics": 0.3459915611814346, "econometrics": 0.2831858407079646, "college_computer_science": 0.3333333333333333, "high_school_biology": 0.36245954692556637, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.3167259786476868, "philosophy": 0.3774193548387097, "professional_medicine": 0.2988929889298893, "nutrition": 0.4163934426229508, "global_facts": 0.3434343434343434, "machine_learning": 0.2972972972972973, "security_studies": 0.29918032786885246, "public_relations": 0.46788990825688076, "professional_psychology": 0.3617021276595745, "prehistory": 0.3993808049535604, "anatomy": 0.4253731343283582, "human_sexuality": 0.45384615384615384, "college_medicine": 0.38372093023255816, "high_school_government_and_politics": 0.4583333333333333, "college_chemistry": 0.2828282828282828, "logical_fallacies": 0.4012345679012346, "high_school_geography": 0.4619289340101523, "elementary_mathematics": 0.30238726790450926, "human_aging": 0.35135135135135137, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.4375, "formal_logic": 0.36, "high_school_statistics": 0.25116279069767444, "international_law": 0.5333333333333333, "high_school_mathematics": 0.22304832713754646, "high_school_computer_science": 0.32323232323232326, "conceptual_physics": 0.3333333333333333, "miscellaneous": 0.4782608695652174, "high_school_chemistry": 0.3069306930693069, "marketing": 0.5708154506437768, "professional_law": 0.299412915851272, "management": 0.45098039215686275, "college_physics": 0.26732673267326734, "jurisprudence": 0.34579439252336447, "world_religions": 0.47058823529411764, "sociology": 0.46, "us_foreign_policy": 0.5757575757575758, "high_school_macroeconomics": 0.3496143958868895, "computer_security": 0.42424242424242425, "moral_scenarios": 0.2360178970917226, "moral_disputes": 0.37681159420289856, "electrical_engineering": 0.3611111111111111, "astronomy": 0.423841059602649, "college_biology": 0.3356643356643357 } }, "prompt_2": { "accuracy": 0.3643189131212013, "category_acc": { "high_school_european_history": 0.4817073170731707, "business_ethics": 0.42424242424242425, "clinical_knowledge": 0.3939393939393939, "medical_genetics": 0.3333333333333333, "high_school_us_history": 0.4630541871921182, "high_school_physics": 0.26, "high_school_world_history": 0.5550847457627118, "virology": 0.3090909090909091, "high_school_microeconomics": 0.34177215189873417, "econometrics": 0.21238938053097345, "college_computer_science": 0.3333333333333333, "high_school_biology": 0.36893203883495146, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.3202846975088968, "philosophy": 0.3903225806451613, "professional_medicine": 0.3247232472324723, "nutrition": 0.4065573770491803, "global_facts": 0.21212121212121213, "machine_learning": 0.2972972972972973, "security_studies": 0.3155737704918033, "public_relations": 0.43119266055045874, "professional_psychology": 0.33387888707037644, "prehistory": 0.38699690402476783, "anatomy": 0.3880597014925373, "human_sexuality": 0.4230769230769231, "college_medicine": 0.3546511627906977, "high_school_government_and_politics": 0.4010416666666667, "college_chemistry": 0.29292929292929293, "logical_fallacies": 0.4074074074074074, "high_school_geography": 0.4416243654822335, "elementary_mathematics": 0.29708222811671087, "human_aging": 0.4009009009009009, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.4227941176470588, "formal_logic": 0.36, "high_school_statistics": 0.2651162790697674, "international_law": 0.49166666666666664, "high_school_mathematics": 0.21561338289962825, "high_school_computer_science": 0.41414141414141414, "conceptual_physics": 0.3162393162393162, "miscellaneous": 0.46547314578005117, "high_school_chemistry": 0.2722772277227723, "marketing": 0.5278969957081545, "professional_law": 0.32615786040443573, "management": 0.5, "college_physics": 0.24752475247524752, "jurisprudence": 0.34579439252336447, "world_religions": 0.4117647058823529, "sociology": 0.5, "us_foreign_policy": 0.6161616161616161, "high_school_macroeconomics": 0.33676092544987146, "computer_security": 0.47474747474747475, "moral_scenarios": 0.26174496644295303, "moral_disputes": 0.36231884057971014, "electrical_engineering": 0.4166666666666667, "astronomy": 0.423841059602649, "college_biology": 0.32867132867132864 } }, "prompt_3": { "accuracy": 0.3584554880228817, "category_acc": { "high_school_european_history": 0.4695121951219512, "business_ethics": 0.43434343434343436, "clinical_knowledge": 0.39015151515151514, "medical_genetics": 0.30303030303030304, "high_school_us_history": 0.4433497536945813, "high_school_physics": 0.26, "high_school_world_history": 0.5296610169491526, "virology": 0.36363636363636365, "high_school_microeconomics": 0.34177215189873417, "econometrics": 0.24778761061946902, "college_computer_science": 0.30303030303030304, "high_school_biology": 0.36893203883495146, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.33451957295373663, "philosophy": 0.3903225806451613, "professional_medicine": 0.3062730627306273, "nutrition": 0.419672131147541, "global_facts": 0.2727272727272727, "machine_learning": 0.3063063063063063, "security_studies": 0.29918032786885246, "public_relations": 0.44036697247706424, "professional_psychology": 0.3469721767594108, "prehistory": 0.4055727554179567, "anatomy": 0.44029850746268656, "human_sexuality": 0.4461538461538462, "college_medicine": 0.3488372093023256, "high_school_government_and_politics": 0.4010416666666667, "college_chemistry": 0.24242424242424243, "logical_fallacies": 0.37037037037037035, "high_school_geography": 0.47715736040609136, "elementary_mathematics": 0.27851458885941643, "human_aging": 0.3918918918918919, "college_mathematics": 0.29292929292929293, "high_school_psychology": 0.4099264705882353, "formal_logic": 0.304, "high_school_statistics": 0.2558139534883721, "international_law": 0.5166666666666667, "high_school_mathematics": 0.2342007434944238, "high_school_computer_science": 0.32323232323232326, "conceptual_physics": 0.2948717948717949, "miscellaneous": 0.4782608695652174, "high_school_chemistry": 0.2722772277227723, "marketing": 0.48497854077253216, "professional_law": 0.30071754729288974, "management": 0.5, "college_physics": 0.25742574257425743, "jurisprudence": 0.3644859813084112, "world_religions": 0.4411764705882353, "sociology": 0.465, "us_foreign_policy": 0.5858585858585859, "high_school_macroeconomics": 0.3059125964010283, "computer_security": 0.37373737373737376, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.3855072463768116, "electrical_engineering": 0.3541666666666667, "astronomy": 0.44370860927152317, "college_biology": 0.36363636363636365 } }, "prompt_4": { "accuracy": 0.3480872363246335, "category_acc": { "high_school_european_history": 0.43902439024390244, "business_ethics": 0.42424242424242425, "clinical_knowledge": 0.4015151515151515, "medical_genetics": 0.32323232323232326, "high_school_us_history": 0.4729064039408867, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.4872881355932203, "virology": 0.2909090909090909, "high_school_microeconomics": 0.31645569620253167, "econometrics": 0.2831858407079646, "college_computer_science": 0.31313131313131315, "high_school_biology": 0.3365695792880259, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.31316725978647686, "philosophy": 0.3548387096774194, "professional_medicine": 0.3062730627306273, "nutrition": 0.4098360655737705, "global_facts": 0.3333333333333333, "machine_learning": 0.3063063063063063, "security_studies": 0.27049180327868855, "public_relations": 0.43119266055045874, "professional_psychology": 0.3404255319148936, "prehistory": 0.38699690402476783, "anatomy": 0.3880597014925373, "human_sexuality": 0.3769230769230769, "college_medicine": 0.4069767441860465, "high_school_government_and_politics": 0.4322916666666667, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.4012345679012346, "high_school_geography": 0.45685279187817257, "elementary_mathematics": 0.2891246684350133, "human_aging": 0.35585585585585583, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.41544117647058826, "formal_logic": 0.344, "high_school_statistics": 0.2558139534883721, "international_law": 0.48333333333333334, "high_school_mathematics": 0.20817843866171004, "high_school_computer_science": 0.32323232323232326, "conceptual_physics": 0.31196581196581197, "miscellaneous": 0.4309462915601023, "high_school_chemistry": 0.2376237623762376, "marketing": 0.48497854077253216, "professional_law": 0.2968036529680365, "management": 0.43137254901960786, "college_physics": 0.2079207920792079, "jurisprudence": 0.308411214953271, "world_religions": 0.4470588235294118, "sociology": 0.405, "us_foreign_policy": 0.5353535353535354, "high_school_macroeconomics": 0.34190231362467866, "computer_security": 0.41414141414141414, "moral_scenarios": 0.22706935123042504, "moral_disputes": 0.36231884057971014, "electrical_engineering": 0.3263888888888889, "astronomy": 0.4370860927152318, "college_biology": 0.32167832167832167 } }, "prompt_5": { "accuracy": 0.35952806578476937, "category_acc": { "high_school_european_history": 0.4268292682926829, "business_ethics": 0.40404040404040403, "clinical_knowledge": 0.375, "medical_genetics": 0.3838383838383838, "high_school_us_history": 0.49261083743842365, "high_school_physics": 0.25333333333333335, "high_school_world_history": 0.4703389830508475, "virology": 0.3212121212121212, "high_school_microeconomics": 0.33755274261603374, "econometrics": 0.23893805309734514, "college_computer_science": 0.26262626262626265, "high_school_biology": 0.37216828478964403, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.3238434163701068, "philosophy": 0.3870967741935484, "professional_medicine": 0.31365313653136534, "nutrition": 0.43278688524590164, "global_facts": 0.26262626262626265, "machine_learning": 0.26126126126126126, "security_studies": 0.30327868852459017, "public_relations": 0.3853211009174312, "professional_psychology": 0.32569558101472995, "prehistory": 0.38390092879256965, "anatomy": 0.43283582089552236, "human_sexuality": 0.4461538461538462, "college_medicine": 0.4127906976744186, "high_school_government_and_politics": 0.4583333333333333, "college_chemistry": 0.25252525252525254, "logical_fallacies": 0.3765432098765432, "high_school_geography": 0.4619289340101523, "elementary_mathematics": 0.3103448275862069, "human_aging": 0.38288288288288286, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.43566176470588236, "formal_logic": 0.264, "high_school_statistics": 0.23255813953488372, "international_law": 0.475, "high_school_mathematics": 0.23048327137546468, "high_school_computer_science": 0.3939393939393939, "conceptual_physics": 0.2777777777777778, "miscellaneous": 0.4846547314578005, "high_school_chemistry": 0.2722772277227723, "marketing": 0.5321888412017167, "professional_law": 0.2974559686888454, "management": 0.5, "college_physics": 0.21782178217821782, "jurisprudence": 0.38317757009345793, "world_religions": 0.40588235294117647, "sociology": 0.495, "us_foreign_policy": 0.6060606060606061, "high_school_macroeconomics": 0.3341902313624679, "computer_security": 0.47474747474747475, "moral_scenarios": 0.24049217002237136, "moral_disputes": 0.39420289855072466, "electrical_engineering": 0.3888888888888889, "astronomy": 0.41721854304635764, "college_biology": 0.3706293706293706 } } }, "c_eval": { "prompt_1": { "accuracy": 0.32540861812778604 }, "prompt_2": { "accuracy": 0.31946508172362553 }, "prompt_3": { "accuracy": 0.3573551263001486 }, "prompt_4": { "accuracy": 0.34843982169390786 }, "prompt_5": { "accuracy": 0.31426448736998513 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.3412204234122042, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.2916666666666667, "computer_architecture": 0.19230769230769232, "college_programming": 0.14285714285714285, "college_physics": 0.2916666666666667, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.09523809523809523, "electrical_engineer": 0.35714285714285715, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.25, "middle_school_mathematics": 0.375, "middle_school_biology": 0.38461538461538464, "middle_school_physics": 0.5, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.4, "business_administration": 0.2631578947368421, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.3448275862068966, "education_science": 0.4411764705882353, "teacher_qualification": 0.3673469387755102, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.4230769230769231, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.375, "logic": 0.48148148148148145, "law": 0.2413793103448276, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.25, "high_school_chinese": 0.16666666666666666, "high_school_history": 0.6, "middle_school_history": 0.37037037037037035, "civil_servant": 0.34615384615384615, "sports_science": 0.3333333333333333, "plant_protection": 0.18518518518518517, "basic_medicine": 0.5, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.4117647058823529, "accountant": 0.35185185185185186, "fire_engineer": 0.3333333333333333, "environmental_impact_assessment_engineer": 0.3055555555555556, "tax_accountant": 0.4074074074074074, "physician": 0.37037037037037035 } }, "prompt_2": { "accuracy": 0.33374844333748444, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.2916666666666667, "computer_architecture": 0.3076923076923077, "college_programming": 0.23809523809523808, "college_physics": 0.20833333333333334, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.125, "middle_school_biology": 0.5, "middle_school_physics": 0.375, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.4166666666666667, "business_administration": 0.39473684210526316, "marxism": 0.375, "mao_zedong_thought": 0.41379310344827586, "education_science": 0.4411764705882353, "teacher_qualification": 0.46938775510204084, "high_school_politics": 0.25, "high_school_geography": 0.25, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.4166666666666667, "logic": 0.37037037037037035, "law": 0.20689655172413793, "chinese_language_and_literature": 0.25, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.35714285714285715, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.48, "middle_school_history": 0.4444444444444444, "civil_servant": 0.28846153846153844, "sports_science": 0.3333333333333333, "plant_protection": 0.37037037037037035, "basic_medicine": 0.5, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.43137254901960786, "accountant": 0.2777777777777778, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.3888888888888889, "physician": 0.4074074074074074 } }, "prompt_3": { "accuracy": 0.3343711083437111, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.25, "computer_architecture": 0.19230769230769232, "college_programming": 0.23809523809523808, "college_physics": 0.375, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.25, "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.4166666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.42857142857142855, "college_economics": 0.4, "business_administration": 0.39473684210526316, "marxism": 0.375, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.5, "teacher_qualification": 0.5102040816326531, "high_school_politics": 0.4166666666666667, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.34615384615384615, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.375, "logic": 0.4074074074074074, "law": 0.3103448275862069, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.39473684210526316, "professional_tour_guide": 0.3235294117647059, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.25, "high_school_history": 0.36, "middle_school_history": 0.37037037037037035, "civil_servant": 0.2692307692307692, "sports_science": 0.375, "plant_protection": 0.4444444444444444, "basic_medicine": 0.5, "clinical_medicine": 0.18518518518518517, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.2777777777777778, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.3333333333333333, "tax_accountant": 0.3148148148148148, "physician": 0.3148148148148148 } }, "prompt_4": { "accuracy": 0.3318804483188045, "category_acc": { "computer_network": 0.5, "operating_system": 0.25, "computer_architecture": 0.19230769230769232, "college_programming": 0.19047619047619047, "college_physics": 0.16666666666666666, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.09523809523809523, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.4827586206896552, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.4166666666666667, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.3076923076923077, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.42857142857142855, "college_economics": 0.3333333333333333, "business_administration": 0.3684210526315789, "marxism": 0.375, "mao_zedong_thought": 0.3793103448275862, "education_science": 0.4411764705882353, "teacher_qualification": 0.42857142857142855, "high_school_politics": 0.375, "high_school_geography": 0.25, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.4166666666666667, "logic": 0.48148148148148145, "law": 0.41379310344827586, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.25, "high_school_chinese": 0.16666666666666666, "high_school_history": 0.52, "middle_school_history": 0.4074074074074074, "civil_servant": 0.25, "sports_science": 0.3333333333333333, "plant_protection": 0.37037037037037035, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.37254901960784315, "accountant": 0.2962962962962963, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3148148148148148, "physician": 0.3888888888888889 } }, "prompt_5": { "accuracy": 0.32814445828144456, "category_acc": { "computer_network": 0.375, "operating_system": 0.375, "computer_architecture": 0.2692307692307692, "college_programming": 0.19047619047619047, "college_physics": 0.3333333333333333, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.25, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.3103448275862069, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.4166666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.25, "middle_school_mathematics": 0.25, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.3333333333333333, "business_administration": 0.23684210526315788, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.3793103448275862, "education_science": 0.5, "teacher_qualification": 0.3469387755102041, "high_school_politics": 0.4583333333333333, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.32142857142857145, "ideological_and_moral_cultivation": 0.25, "logic": 0.37037037037037035, "law": 0.3103448275862069, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.4, "middle_school_history": 0.48148148148148145, "civil_servant": 0.34615384615384615, "sports_science": 0.2916666666666667, "plant_protection": 0.25925925925925924, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.39215686274509803, "accountant": 0.3888888888888889, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.3333333333333333, "tax_accountant": 0.37037037037037035, "physician": 0.42592592592592593 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.3978494623655914 }, "prompt_2": { "accuracy": 0.3727598566308244 }, "prompt_3": { "accuracy": 0.34050179211469533 }, "prompt_4": { "accuracy": 0.3548387096774194 }, "prompt_5": { "accuracy": 0.3118279569892473 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.3173027111034364, "category_acc": { "agronomy": 0.3136094674556213, "anatomy": 0.30405405405405406, "ancient_chinese": 0.3170731707317073, "arts": 0.39375, "astronomy": 0.2727272727272727, "business_ethics": 0.3062200956937799, "chinese_civil_service_exam": 0.3125, "chinese_driving_rule": 0.4351145038167939, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.38317757009345793, "chinese_history": 0.33436532507739936, "chinese_literature": 0.3137254901960784, "chinese_teacher_qualification": 0.36312849162011174, "clinical_knowledge": 0.2869198312236287, "college_actuarial_science": 0.2830188679245283, "college_education": 0.29906542056074764, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.2222222222222222, "college_mathematics": 0.3238095238095238, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.32967032967032966, "computer_science": 0.28921568627450983, "computer_security": 0.3684210526315789, "conceptual_physics": 0.32653061224489793, "construction_project_management": 0.2446043165467626, "economics": 0.3584905660377358, "education": 0.3987730061349693, "electrical_engineering": 0.29069767441860467, "elementary_chinese": 0.27380952380952384, "elementary_commonsense": 0.29292929292929293, "elementary_information_and_technology": 0.3907563025210084, "elementary_mathematics": 0.26521739130434785, "ethnology": 0.2814814814814815, "food_science": 0.3146853146853147, "genetics": 0.2556818181818182, "global_facts": 0.3422818791946309, "high_school_biology": 0.23668639053254437, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.21341463414634146, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.4125874125874126, "human_sexuality": 0.30952380952380953, "international_law": 0.2648648648648649, "journalism": 0.3488372093023256, "jurisprudence": 0.340632603406326, "legal_and_moral_basis": 0.514018691588785, "logical": 0.2926829268292683, "machine_learning": 0.2459016393442623, "management": 0.3238095238095238, "marketing": 0.4222222222222222, "marxist_theory": 0.37566137566137564, "modern_chinese": 0.27586206896551724, "nutrition": 0.31724137931034485, "philosophy": 0.3619047619047619, "professional_accounting": 0.33714285714285713, "professional_law": 0.2796208530805687, "professional_medicine": 0.2579787234042553, "professional_psychology": 0.33620689655172414, "public_relations": 0.3505747126436782, "security_study": 0.32592592592592595, "sociology": 0.33185840707964603, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.2648648648648649, "virology": 0.3905325443786982, "world_history": 0.3416149068322981, "world_religions": 0.33125 } }, "prompt_2": { "accuracy": 0.32274218615092387, "category_acc": { "agronomy": 0.3905325443786982, "anatomy": 0.30405405405405406, "ancient_chinese": 0.2621951219512195, "arts": 0.4375, "astronomy": 0.2787878787878788, "business_ethics": 0.3923444976076555, "chinese_civil_service_exam": 0.31875, "chinese_driving_rule": 0.40458015267175573, "chinese_food_culture": 0.27941176470588236, "chinese_foreign_policy": 0.411214953271028, "chinese_history": 0.38699690402476783, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.40782122905027934, "clinical_knowledge": 0.3291139240506329, "college_actuarial_science": 0.29245283018867924, "college_education": 0.3364485981308411, "college_engineering_hydrology": 0.36792452830188677, "college_law": 0.28703703703703703, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.27358490566037735, "college_medicine": 0.2783882783882784, "computer_science": 0.2647058823529412, "computer_security": 0.32748538011695905, "conceptual_physics": 0.2925170068027211, "construction_project_management": 0.28776978417266186, "economics": 0.3522012578616352, "education": 0.37423312883435583, "electrical_engineering": 0.31976744186046513, "elementary_chinese": 0.34523809523809523, "elementary_commonsense": 0.2676767676767677, "elementary_information_and_technology": 0.3235294117647059, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.2962962962962963, "food_science": 0.22377622377622378, "genetics": 0.26136363636363635, "global_facts": 0.31543624161073824, "high_school_biology": 0.28994082840236685, "high_school_chemistry": 0.23484848484848486, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.36363636363636365, "human_sexuality": 0.3412698412698413, "international_law": 0.25405405405405407, "journalism": 0.38372093023255816, "jurisprudence": 0.35279805352798055, "legal_and_moral_basis": 0.5046728971962616, "logical": 0.2845528455284553, "machine_learning": 0.29508196721311475, "management": 0.3476190476190476, "marketing": 0.37222222222222223, "marxist_theory": 0.3968253968253968, "modern_chinese": 0.28448275862068967, "nutrition": 0.31724137931034485, "philosophy": 0.34285714285714286, "professional_accounting": 0.30857142857142855, "professional_law": 0.27488151658767773, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.34913793103448276, "public_relations": 0.3275862068965517, "security_study": 0.34814814814814815, "sociology": 0.30973451327433627, "sports_science": 0.2545454545454545, "traditional_chinese_medicine": 0.2864864864864865, "virology": 0.2958579881656805, "world_history": 0.45962732919254656, "world_religions": 0.3625 } }, "prompt_3": { "accuracy": 0.3358659989639095, "category_acc": { "agronomy": 0.35502958579881655, "anatomy": 0.2905405405405405, "ancient_chinese": 0.23170731707317074, "arts": 0.4, "astronomy": 0.2909090909090909, "business_ethics": 0.36363636363636365, "chinese_civil_service_exam": 0.33125, "chinese_driving_rule": 0.4198473282442748, "chinese_food_culture": 0.3088235294117647, "chinese_foreign_policy": 0.32710280373831774, "chinese_history": 0.3591331269349845, "chinese_literature": 0.27941176470588236, "chinese_teacher_qualification": 0.37988826815642457, "clinical_knowledge": 0.3291139240506329, "college_actuarial_science": 0.3018867924528302, "college_education": 0.32710280373831774, "college_engineering_hydrology": 0.37735849056603776, "college_law": 0.3148148148148148, "college_mathematics": 0.23809523809523808, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.34065934065934067, "computer_science": 0.3088235294117647, "computer_security": 0.36257309941520466, "conceptual_physics": 0.30612244897959184, "construction_project_management": 0.3237410071942446, "economics": 0.36477987421383645, "education": 0.4171779141104294, "electrical_engineering": 0.31976744186046513, "elementary_chinese": 0.3134920634920635, "elementary_commonsense": 0.2878787878787879, "elementary_information_and_technology": 0.42436974789915966, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.3111111111111111, "food_science": 0.2867132867132867, "genetics": 0.26704545454545453, "global_facts": 0.3624161073825503, "high_school_biology": 0.26627218934911245, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.3644067796610169, "high_school_mathematics": 0.24390243902439024, "high_school_physics": 0.3, "high_school_politics": 0.3916083916083916, "human_sexuality": 0.3492063492063492, "international_law": 0.35135135135135137, "journalism": 0.4127906976744186, "jurisprudence": 0.35523114355231145, "legal_and_moral_basis": 0.5327102803738317, "logical": 0.3170731707317073, "machine_learning": 0.32786885245901637, "management": 0.3523809523809524, "marketing": 0.3888888888888889, "marxist_theory": 0.3862433862433862, "modern_chinese": 0.29310344827586204, "nutrition": 0.3448275862068966, "philosophy": 0.3904761904761905, "professional_accounting": 0.33714285714285713, "professional_law": 0.2938388625592417, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.3620689655172414, "public_relations": 0.39655172413793105, "security_study": 0.34074074074074073, "sociology": 0.3584070796460177, "sports_science": 0.3151515151515151, "traditional_chinese_medicine": 0.3081081081081081, "virology": 0.2958579881656805, "world_history": 0.40372670807453415, "world_religions": 0.40625 } }, "prompt_4": { "accuracy": 0.33223968226558453, "category_acc": { "agronomy": 0.3254437869822485, "anatomy": 0.30405405405405406, "ancient_chinese": 0.20121951219512196, "arts": 0.40625, "astronomy": 0.24848484848484848, "business_ethics": 0.3253588516746411, "chinese_civil_service_exam": 0.29375, "chinese_driving_rule": 0.4198473282442748, "chinese_food_culture": 0.2867647058823529, "chinese_foreign_policy": 0.38317757009345793, "chinese_history": 0.33746130030959753, "chinese_literature": 0.31862745098039214, "chinese_teacher_qualification": 0.40782122905027934, "clinical_knowledge": 0.3080168776371308, "college_actuarial_science": 0.3018867924528302, "college_education": 0.34579439252336447, "college_engineering_hydrology": 0.39622641509433965, "college_law": 0.25925925925925924, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.2641509433962264, "college_medicine": 0.32967032967032966, "computer_science": 0.3333333333333333, "computer_security": 0.32748538011695905, "conceptual_physics": 0.3197278911564626, "construction_project_management": 0.38848920863309355, "economics": 0.3333333333333333, "education": 0.39263803680981596, "electrical_engineering": 0.3430232558139535, "elementary_chinese": 0.3253968253968254, "elementary_commonsense": 0.32323232323232326, "elementary_information_and_technology": 0.40336134453781514, "elementary_mathematics": 0.25217391304347825, "ethnology": 0.2740740740740741, "food_science": 0.2867132867132867, "genetics": 0.3125, "global_facts": 0.31543624161073824, "high_school_biology": 0.25443786982248523, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.2542372881355932, "high_school_mathematics": 0.23170731707317074, "high_school_physics": 0.21818181818181817, "high_school_politics": 0.3986013986013986, "human_sexuality": 0.3333333333333333, "international_law": 0.32972972972972975, "journalism": 0.38372093023255816, "jurisprudence": 0.30900243309002434, "legal_and_moral_basis": 0.6214953271028038, "logical": 0.34146341463414637, "machine_learning": 0.23770491803278687, "management": 0.35714285714285715, "marketing": 0.4111111111111111, "marxist_theory": 0.38095238095238093, "modern_chinese": 0.3017241379310345, "nutrition": 0.35172413793103446, "philosophy": 0.3523809523809524, "professional_accounting": 0.3485714285714286, "professional_law": 0.27488151658767773, "professional_medicine": 0.26063829787234044, "professional_psychology": 0.375, "public_relations": 0.3563218390804598, "security_study": 0.3851851851851852, "sociology": 0.39823008849557523, "sports_science": 0.3212121212121212, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.3254437869822485, "world_history": 0.40993788819875776, "world_religions": 0.43125 } }, "prompt_5": { "accuracy": 0.3084959419789328, "category_acc": { "agronomy": 0.2603550295857988, "anatomy": 0.32432432432432434, "ancient_chinese": 0.2865853658536585, "arts": 0.34375, "astronomy": 0.23030303030303031, "business_ethics": 0.3444976076555024, "chinese_civil_service_exam": 0.36875, "chinese_driving_rule": 0.4351145038167939, "chinese_food_culture": 0.21323529411764705, "chinese_foreign_policy": 0.38317757009345793, "chinese_history": 0.3653250773993808, "chinese_literature": 0.3088235294117647, "chinese_teacher_qualification": 0.31843575418994413, "clinical_knowledge": 0.2911392405063291, "college_actuarial_science": 0.33962264150943394, "college_education": 0.37383177570093457, "college_engineering_hydrology": 0.2830188679245283, "college_law": 0.2777777777777778, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2967032967032967, "computer_science": 0.29411764705882354, "computer_security": 0.30409356725146197, "conceptual_physics": 0.2925170068027211, "construction_project_management": 0.302158273381295, "economics": 0.3710691823899371, "education": 0.3803680981595092, "electrical_engineering": 0.26744186046511625, "elementary_chinese": 0.26587301587301587, "elementary_commonsense": 0.2878787878787879, "elementary_information_and_technology": 0.35294117647058826, "elementary_mathematics": 0.26521739130434785, "ethnology": 0.31851851851851853, "food_science": 0.3006993006993007, "genetics": 0.2727272727272727, "global_facts": 0.3959731543624161, "high_school_biology": 0.23076923076923078, "high_school_chemistry": 0.17424242424242425, "high_school_geography": 0.211864406779661, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.40559440559440557, "human_sexuality": 0.29365079365079366, "international_law": 0.2864864864864865, "journalism": 0.3430232558139535, "jurisprudence": 0.2798053527980535, "legal_and_moral_basis": 0.5046728971962616, "logical": 0.3089430894308943, "machine_learning": 0.20491803278688525, "management": 0.319047619047619, "marketing": 0.3388888888888889, "marxist_theory": 0.3439153439153439, "modern_chinese": 0.3448275862068966, "nutrition": 0.3310344827586207, "philosophy": 0.3047619047619048, "professional_accounting": 0.26857142857142857, "professional_law": 0.27014218009478674, "professional_medicine": 0.2526595744680851, "professional_psychology": 0.3103448275862069, "public_relations": 0.3160919540229885, "security_study": 0.34074074074074073, "sociology": 0.3053097345132743, "sports_science": 0.28484848484848485, "traditional_chinese_medicine": 0.2756756756756757, "virology": 0.3254437869822485, "world_history": 0.391304347826087, "world_religions": 0.34375 } } }, "zbench": { "prompt_1": { "accuracy": 0.48484848484848486 }, "prompt_2": { "accuracy": 0.42424242424242425 }, "prompt_3": { "accuracy": 0.42424242424242425 }, "prompt_4": { "accuracy": 0.36363636363636365 }, "prompt_5": { "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.5181818181818182 }, "prompt_2": { "accuracy": 0.44545454545454544 }, "prompt_3": { "accuracy": 0.42045454545454547 }, "prompt_4": { "accuracy": 0.43636363636363634 }, "prompt_5": { "accuracy": 0.41818181818181815 } }, "ocnli": { "prompt_1": { "accuracy": 0.31864406779661014 }, "prompt_2": { "accuracy": 0.31796610169491524 }, "prompt_3": { "accuracy": 0.32033898305084746 }, "prompt_4": { "accuracy": 0.3254237288135593 }, "prompt_5": { "accuracy": 0.32440677966101694 } }, "c3": { "prompt_1": { "accuracy": 0.5474943904263276 }, "prompt_2": { "accuracy": 0.5381451009723262 }, "prompt_3": { "accuracy": 0.5138369483919222 }, "prompt_4": { "accuracy": 0.524682124158564 }, "prompt_5": { "accuracy": 0.5224382946896036 } }, "dream": { "prompt_1": { "accuracy": 0.5972562469377756 }, "prompt_2": { "accuracy": 0.6438020578147967 }, "prompt_3": { "accuracy": 0.6266536011758942 }, "prompt_4": { "accuracy": 0.5658990690837825 }, "prompt_5": { "accuracy": 0.6153846153846154 } }, "samsum": { "prompt_1": { "rouge1": 0.19421979959087435, "rouge2": 0.06983749460043516, "rougeL": 0.168321567036734, "avg_rouge": 0.14412628707601452 }, "prompt_2": { "rouge1": 0.19866330633085436, "rouge2": 0.07228786591276251, "rougeL": 0.17140262631047645, "avg_rouge": 0.14745126618469775 }, "prompt_3": { "rouge1": 0.1823010767794734, "rouge2": 0.062186205910091226, "rougeL": 0.1637899495200591, "avg_rouge": 0.13609241073654124 }, "prompt_4": { "rouge1": 0.19574582248288472, "rouge2": 0.07082506606400779, "rougeL": 0.16926419301155568, "avg_rouge": 0.14527836051948273 }, "prompt_5": { "rouge1": 0.19145248800250056, "rouge2": 0.06645109075593836, "rougeL": 0.16682834576905436, "avg_rouge": 0.1415773081758311 } }, "dialogsum": { "prompt_1": { "rouge1": 0.19354612820856404, "rouge2": 0.05220115551483978, "rougeL": 0.1475132846001137, "avg_rouge": 0.13108685610783918 }, "prompt_2": { "rouge1": 0.18930219636318057, "rouge2": 0.0509957241945143, "rougeL": 0.14481650495763934, "avg_rouge": 0.12837147517177808 }, "prompt_3": { "rouge1": 0.18856305511927057, "rouge2": 0.05047203018596512, "rougeL": 0.14516669076855662, "avg_rouge": 0.12806725869126412 }, "prompt_4": { "rouge1": 0.19987887059378412, "rouge2": 0.054264246884584, "rougeL": 0.15137121232130749, "avg_rouge": 0.13517144326655853 }, "prompt_5": { "rouge1": 0.1887689700181949, "rouge2": 0.050424274072061896, "rougeL": 0.14477589018893963, "avg_rouge": 0.1279897114263988 } }, "sst2": { "prompt_1": { "accuracy": 0.8142201834862385 }, "prompt_2": { "accuracy": 0.7889908256880734 }, "prompt_3": { "accuracy": 0.731651376146789 }, "prompt_4": { "accuracy": 0.7763761467889908 }, "prompt_5": { "accuracy": 0.8027522935779816 } }, "cola": { "prompt_1": { "accuracy": 0.5589645254074784 }, "prompt_2": { "accuracy": 0.5100671140939598 }, "prompt_3": { "accuracy": 0.5982742090124641 }, "prompt_4": { "accuracy": 0.5973154362416108 }, "prompt_5": { "accuracy": 0.6883988494726749 } }, "qqp": { "prompt_1": { "accuracy": 0.4865 }, "prompt_2": { "accuracy": 0.4955 }, "prompt_3": { "accuracy": 0.507 }, "prompt_4": { "accuracy": 0.4535 }, "prompt_5": { "accuracy": 0.4795 } }, "mnli": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.5 } }, "qnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.2 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } }, "rte": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.5 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.8 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "sailor_4b_chat": { "model_size": "4B", "model_link": "https://huggingface.co/sail/Sailor-4B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.859873949579832, "language_acc": { "Spanish": 0.853781512605042, "English": 0.8823529411764706, "Chinese": 0.8487394957983193, "Vietnamese": 0.8546218487394958 }, "consistency_score_2": 0.8463585434173669, "consistency_score_3": 0.7785714285714285, "consistency_score_4": 0.7344537815126051, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8689075630252101, "Spanish,Chinese": 0.8403361344537815, "Spanish,Vietnamese": 0.846218487394958, "English,Chinese": 0.8495798319327731, "English,Vietnamese": 0.846218487394958, "Chinese,Vietnamese": 0.826890756302521 }, "3_combine": { "Spanish,English,Chinese": 0.788235294117647, "Spanish,English,Vietnamese": 0.7890756302521008, "Spanish,Chinese,Vietnamese": 0.7655462184873949, "English,Chinese,Vietnamese": 0.7714285714285715 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7344537815126051 } }, "AC3_2": 0.8530627173414949, "AC3_3": 0.8172055024870784, "AC3_4": 0.7922306832369328 }, "prompt_2": { "overall_acc": 0.876890756302521, "language_acc": { "Spanish": 0.8747899159663866, "English": 0.8932773109243698, "Chinese": 0.8663865546218488, "Vietnamese": 0.873109243697479 }, "consistency_score_2": 0.8628851540616246, "consistency_score_3": 0.8012605042016806, "consistency_score_4": 0.7588235294117647, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8966386554621849, "Spanish,Chinese": 0.8571428571428571, "Spanish,Vietnamese": 0.8546218487394958, "English,Chinese": 0.8697478991596639, "English,Vietnamese": 0.8579831932773109, "Chinese,Vietnamese": 0.8411764705882353 }, "3_combine": { "Spanish,English,Chinese": 0.8159663865546218, "Spanish,English,Vietnamese": 0.8117647058823529, "Spanish,Chinese,Vietnamese": 0.7840336134453781, "English,Chinese,Vietnamese": 0.7932773109243697 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7588235294117647 } }, "AC3_2": 0.869831580948205, "AC3_3": 0.8373713931743918, "AC3_4": 0.8135960471541407 }, "prompt_3": { "overall_acc": 0.8745798319327731, "language_acc": { "Spanish": 0.8714285714285714, "English": 0.892436974789916, "Chinese": 0.8697478991596639, "Vietnamese": 0.8647058823529412 }, "consistency_score_2": 0.8641456582633054, "consistency_score_3": 0.8033613445378152, "consistency_score_4": 0.7630252100840336, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8865546218487395, "Spanish,Chinese": 0.8579831932773109, "Spanish,Vietnamese": 0.8596638655462185, "English,Chinese": 0.8705882352941177, "English,Vietnamese": 0.8630252100840337, "Chinese,Vietnamese": 0.8470588235294118 }, "3_combine": { "Spanish,English,Chinese": 0.8126050420168067, "Spanish,English,Vietnamese": 0.8117647058823529, "Spanish,Chinese,Vietnamese": 0.7907563025210084, "English,Chinese,Vietnamese": 0.7983193277310925 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7630252100840336 } }, "AC3_2": 0.8693314370638944, "AC3_3": 0.8374591904624037, "AC3_4": 0.8150029376230561 }, "prompt_4": { "overall_acc": 0.8657563025210084, "language_acc": { "Spanish": 0.8554621848739495, "English": 0.8890756302521008, "Chinese": 0.8512605042016806, "Vietnamese": 0.8672268907563025 }, "consistency_score_2": 0.8463585434173669, "consistency_score_3": 0.7800420168067226, "consistency_score_4": 0.7361344537815127, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.880672268907563, "Spanish,Chinese": 0.8302521008403362, "Spanish,Vietnamese": 0.846218487394958, "English,Chinese": 0.8512605042016806, "English,Vietnamese": 0.8512605042016806, "Chinese,Vietnamese": 0.8184873949579832 }, "3_combine": { "Spanish,English,Chinese": 0.7924369747899159, "Spanish,English,Vietnamese": 0.7966386554621848, "Spanish,Chinese,Vietnamese": 0.7588235294117647, "English,Chinese,Vietnamese": 0.7722689075630252 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7361344537815127 } }, "AC3_2": 0.8559475374581995, "AC3_3": 0.8206671307289214, "AC3_4": 0.7957010055984883 }, "prompt_5": { "overall_acc": 0.8590336134453782, "language_acc": { "Spanish": 0.8369747899159664, "English": 0.8831932773109243, "Chinese": 0.8546218487394958, "Vietnamese": 0.8613445378151261 }, "consistency_score_2": 0.8570028011204482, "consistency_score_3": 0.7932773109243697, "consistency_score_4": 0.7504201680672269, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8714285714285714, "Spanish,Chinese": 0.8470588235294118, "Spanish,Vietnamese": 0.8596638655462185, "English,Chinese": 0.8714285714285714, "English,Vietnamese": 0.8638655462184874, "Chinese,Vietnamese": 0.8285714285714286 }, "3_combine": { "Spanish,English,Chinese": 0.8016806722689076, "Spanish,English,Vietnamese": 0.8058823529411765, "Spanish,Chinese,Vietnamese": 0.7756302521008404, "English,Chinese,Vietnamese": 0.7899159663865546 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7504201680672269 } }, "AC3_2": 0.8580170055688157, "AC3_3": 0.8248470245832001, "AC3_4": 0.8010620198502123 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.48000000000000004, "language_acc": { "Filipino": 0.36666666666666664, "Vietnamese": 0.44666666666666666, "Chinese": 0.5133333333333333, "Spanish": 0.4533333333333333, "Malay": 0.5333333333333333, "Indonesian": 0.48, "English": 0.5666666666666667 }, "consistency_score_2": 0.6336507936507936, "consistency_score_3": 0.49371428571428566, "consistency_score_4": 0.41638095238095235, "consistency_score_5": 0.36634920634920637, "consistency_score_6": 0.3314285714285714, "consistency_score_7": 0.30666666666666664, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.54, "Filipino,Chinese": 0.47333333333333333, "Filipino,Spanish": 0.5533333333333333, "Filipino,Malay": 0.5666666666666667, "Filipino,Indonesian": 0.5, "Filipino,English": 0.5666666666666667, "Vietnamese,Chinese": 0.6333333333333333, "Vietnamese,Spanish": 0.62, "Vietnamese,Malay": 0.68, "Vietnamese,Indonesian": 0.66, "Vietnamese,English": 0.68, "Chinese,Spanish": 0.6133333333333333, "Chinese,Malay": 0.6466666666666666, "Chinese,Indonesian": 0.66, "Chinese,English": 0.7133333333333334, "Spanish,Malay": 0.6533333333333333, "Spanish,Indonesian": 0.6666666666666666, "Spanish,English": 0.6866666666666666, "Malay,Indonesian": 0.72, "Malay,English": 0.76, "Indonesian,English": 0.7133333333333334 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.3933333333333333, "Filipino,Vietnamese,Spanish": 0.41333333333333333, "Filipino,Vietnamese,Malay": 0.4533333333333333, "Filipino,Vietnamese,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,English": 0.44666666666666666, "Filipino,Chinese,Spanish": 0.38666666666666666, "Filipino,Chinese,Malay": 0.4, "Filipino,Chinese,Indonesian": 0.37333333333333335, "Filipino,Chinese,English": 0.4266666666666667, "Filipino,Spanish,Malay": 0.4533333333333333, "Filipino,Spanish,Indonesian": 0.4066666666666667, "Filipino,Spanish,English": 0.4533333333333333, "Filipino,Malay,Indonesian": 0.44666666666666666, "Filipino,Malay,English": 0.4866666666666667, "Filipino,Indonesian,English": 0.4266666666666667, "Vietnamese,Chinese,Spanish": 0.48, "Vietnamese,Chinese,Malay": 0.5266666666666666, "Vietnamese,Chinese,Indonesian": 0.5133333333333333, "Vietnamese,Chinese,English": 0.5466666666666666, "Vietnamese,Spanish,Malay": 0.52, "Vietnamese,Spanish,Indonesian": 0.5133333333333333, "Vietnamese,Spanish,English": 0.5133333333333333, "Vietnamese,Malay,Indonesian": 0.58, "Vietnamese,Malay,English": 0.6, "Vietnamese,Indonesian,English": 0.5466666666666666, "Chinese,Spanish,Malay": 0.5, "Chinese,Spanish,Indonesian": 0.5133333333333333, "Chinese,Spanish,English": 0.5266666666666666, "Chinese,Malay,Indonesian": 0.5466666666666666, "Chinese,Malay,English": 0.6, "Chinese,Indonesian,English": 0.5733333333333334, "Spanish,Malay,Indonesian": 0.54, "Spanish,Malay,English": 0.58, "Spanish,Indonesian,English": 0.56, "Malay,Indonesian,English": 0.62 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.3466666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.36, "Filipino,Vietnamese,Chinese,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Chinese,English": 0.37333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.38666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Spanish,English": 0.38, "Filipino,Vietnamese,Malay,Indonesian": 0.38666666666666666, "Filipino,Vietnamese,Malay,English": 0.4066666666666667, "Filipino,Vietnamese,Indonesian,English": 0.36666666666666664, "Filipino,Chinese,Spanish,Malay": 0.35333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.3466666666666667, "Filipino,Chinese,Spanish,English": 0.36666666666666664, "Filipino,Chinese,Malay,Indonesian": 0.36, "Filipino,Chinese,Malay,English": 0.3933333333333333, "Filipino,Chinese,Indonesian,English": 0.36, "Filipino,Spanish,Malay,Indonesian": 0.38666666666666666, "Filipino,Spanish,Malay,English": 0.42, "Filipino,Spanish,Indonesian,English": 0.38, "Filipino,Malay,Indonesian,English": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.4266666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.42, "Vietnamese,Chinese,Spanish,English": 0.43333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.48, "Vietnamese,Chinese,Malay,English": 0.5066666666666667, "Vietnamese,Chinese,Indonesian,English": 0.47333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Spanish,Malay,English": 0.48, "Vietnamese,Spanish,Indonesian,English": 0.44666666666666666, "Vietnamese,Malay,Indonesian,English": 0.5333333333333333, "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Chinese,Spanish,Malay,English": 0.47333333333333333, "Chinese,Spanish,Indonesian,English": 0.4666666666666667, "Chinese,Malay,Indonesian,English": 0.52, "Spanish,Malay,Indonesian,English": 0.5066666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.36, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.34, "Filipino,Vietnamese,Malay,Indonesian,English": 0.36, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.3333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.3466666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.3333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.35333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.36666666666666664, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3933333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.4666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.44, "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.32, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.30666666666666664 } }, "AC3_2": 0.5462257696202791, "AC3_3": 0.48676056333029155, "AC3_4": 0.4459328516289599, "AC3_5": 0.4155438859223945, "AC3_6": 0.39211267600801425, "AC3_7": 0.3742372880880207 }, "prompt_2": { "overall_acc": 0.48, "language_acc": { "Filipino": 0.36, "Vietnamese": 0.4666666666666667, "Chinese": 0.49333333333333335, "Spanish": 0.4666666666666667, "Malay": 0.5333333333333333, "Indonesian": 0.48, "English": 0.56 }, "consistency_score_2": 0.6298412698412699, "consistency_score_3": 0.48876190476190484, "consistency_score_4": 0.41066666666666674, "consistency_score_5": 0.36000000000000004, "consistency_score_6": 0.3247619047619047, "consistency_score_7": 0.3, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5, "Filipino,Chinese": 0.5066666666666667, "Filipino,Spanish": 0.52, "Filipino,Malay": 0.5266666666666666, "Filipino,Indonesian": 0.52, "Filipino,English": 0.56, "Vietnamese,Chinese": 0.6266666666666667, "Vietnamese,Spanish": 0.5533333333333333, "Vietnamese,Malay": 0.68, "Vietnamese,Indonesian": 0.6866666666666666, "Vietnamese,English": 0.6933333333333334, "Chinese,Spanish": 0.58, "Chinese,Malay": 0.6866666666666666, "Chinese,Indonesian": 0.68, "Chinese,English": 0.7533333333333333, "Spanish,Malay": 0.6133333333333333, "Spanish,Indonesian": 0.64, "Spanish,English": 0.6666666666666666, "Malay,Indonesian": 0.7533333333333333, "Malay,English": 0.72, "Indonesian,English": 0.76 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.3933333333333333, "Filipino,Vietnamese,Spanish": 0.36666666666666664, "Filipino,Vietnamese,Malay": 0.41333333333333333, "Filipino,Vietnamese,Indonesian": 0.4066666666666667, "Filipino,Vietnamese,English": 0.41333333333333333, "Filipino,Chinese,Spanish": 0.37333333333333335, "Filipino,Chinese,Malay": 0.41333333333333333, "Filipino,Chinese,Indonesian": 0.3933333333333333, "Filipino,Chinese,English": 0.44666666666666666, "Filipino,Spanish,Malay": 0.3933333333333333, "Filipino,Spanish,Indonesian": 0.4066666666666667, "Filipino,Spanish,English": 0.43333333333333335, "Filipino,Malay,Indonesian": 0.44666666666666666, "Filipino,Malay,English": 0.4533333333333333, "Filipino,Indonesian,English": 0.44666666666666666, "Vietnamese,Chinese,Spanish": 0.44, "Vietnamese,Chinese,Malay": 0.54, "Vietnamese,Chinese,Indonesian": 0.5266666666666666, "Vietnamese,Chinese,English": 0.5666666666666667, "Vietnamese,Spanish,Malay": 0.4866666666666667, "Vietnamese,Spanish,Indonesian": 0.49333333333333335, "Vietnamese,Spanish,English": 0.49333333333333335, "Vietnamese,Malay,Indonesian": 0.6066666666666667, "Vietnamese,Malay,English": 0.6, "Vietnamese,Indonesian,English": 0.5933333333333334, "Chinese,Spanish,Malay": 0.4866666666666667, "Chinese,Spanish,Indonesian": 0.4866666666666667, "Chinese,Spanish,English": 0.5266666666666666, "Chinese,Malay,Indonesian": 0.58, "Chinese,Malay,English": 0.6066666666666667, "Chinese,Indonesian,English": 0.62, "Spanish,Malay,Indonesian": 0.5333333333333333, "Spanish,Malay,English": 0.5333333333333333, "Spanish,Indonesian,English": 0.5533333333333333, "Malay,Indonesian,English": 0.6333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.36666666666666664, "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Chinese,English": 0.37333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.3333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.34, "Filipino,Vietnamese,Spanish,English": 0.34, "Filipino,Vietnamese,Malay,Indonesian": 0.38, "Filipino,Vietnamese,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.37333333333333335, "Filipino,Chinese,Spanish,Malay": 0.34, "Filipino,Chinese,Spanish,Indonesian": 0.3466666666666667, "Filipino,Chinese,Spanish,English": 0.36, "Filipino,Chinese,Malay,Indonesian": 0.37333333333333335, "Filipino,Chinese,Malay,English": 0.3933333333333333, "Filipino,Chinese,Indonesian,English": 0.38, "Filipino,Spanish,Malay,Indonesian": 0.36666666666666664, "Filipino,Spanish,Malay,English": 0.36666666666666664, "Filipino,Spanish,Indonesian,English": 0.38666666666666666, "Filipino,Malay,Indonesian,English": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.42, "Vietnamese,Chinese,Spanish,Indonesian": 0.4066666666666667, "Vietnamese,Chinese,Spanish,English": 0.43333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.49333333333333335, "Vietnamese,Chinese,Malay,English": 0.5133333333333333, "Vietnamese,Chinese,Indonesian,English": 0.5, "Vietnamese,Spanish,Malay,Indonesian": 0.46, "Vietnamese,Spanish,Malay,English": 0.46, "Vietnamese,Spanish,Indonesian,English": 0.4533333333333333, "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay,Indonesian": 0.44, "Chinese,Spanish,Malay,English": 0.4666666666666667, "Chinese,Spanish,Indonesian,English": 0.4666666666666667, "Chinese,Malay,Indonesian,English": 0.5333333333333333, "Spanish,Malay,Indonesian,English": 0.48 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Spanish,English": 0.32, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.32, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.32666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.36, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.3333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.34, "Filipino,Chinese,Malay,Indonesian,English": 0.36, "Filipino,Spanish,Malay,Indonesian,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3933333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4, "Vietnamese,Chinese,Malay,Indonesian,English": 0.47333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.43333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.42 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.3333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.31333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.32, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3 } }, "AC3_2": 0.5448054919417581, "AC3_3": 0.4843413290888165, "AC3_4": 0.44263473048922514, "AC3_5": 0.4114285713795918, "AC3_6": 0.38740828397552907, "AC3_7": 0.3692307691834319 }, "prompt_3": { "overall_acc": 0.48761904761904756, "language_acc": { "Filipino": 0.3333333333333333, "Vietnamese": 0.49333333333333335, "Chinese": 0.49333333333333335, "Spanish": 0.47333333333333333, "Malay": 0.5066666666666667, "Indonesian": 0.5466666666666666, "English": 0.5666666666666667 }, "consistency_score_2": 0.633015873015873, "consistency_score_3": 0.49161904761904773, "consistency_score_4": 0.4108571428571428, "consistency_score_5": 0.3558730158730159, "consistency_score_6": 0.31428571428571433, "consistency_score_7": 0.28, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.47333333333333333, "Filipino,Chinese": 0.5466666666666666, "Filipino,Spanish": 0.4666666666666667, "Filipino,Malay": 0.5333333333333333, "Filipino,Indonesian": 0.5266666666666666, "Filipino,English": 0.54, "Vietnamese,Chinese": 0.6466666666666666, "Vietnamese,Spanish": 0.64, "Vietnamese,Malay": 0.6666666666666666, "Vietnamese,Indonesian": 0.6866666666666666, "Vietnamese,English": 0.6933333333333334, "Chinese,Spanish": 0.6333333333333333, "Chinese,Malay": 0.66, "Chinese,Indonesian": 0.6733333333333333, "Chinese,English": 0.7333333333333333, "Spanish,Malay": 0.6266666666666667, "Spanish,Indonesian": 0.7, "Spanish,English": 0.6933333333333334, "Malay,Indonesian": 0.74, "Malay,English": 0.6866666666666666, "Indonesian,English": 0.7266666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.4066666666666667, "Filipino,Vietnamese,Spanish": 0.3466666666666667, "Filipino,Vietnamese,Malay": 0.4, "Filipino,Vietnamese,Indonesian": 0.3933333333333333, "Filipino,Vietnamese,English": 0.4, "Filipino,Chinese,Spanish": 0.38666666666666666, "Filipino,Chinese,Malay": 0.42, "Filipino,Chinese,Indonesian": 0.42, "Filipino,Chinese,English": 0.4533333333333333, "Filipino,Spanish,Malay": 0.38, "Filipino,Spanish,Indonesian": 0.38666666666666666, "Filipino,Spanish,English": 0.3933333333333333, "Filipino,Malay,Indonesian": 0.44, "Filipino,Malay,English": 0.43333333333333335, "Filipino,Indonesian,English": 0.44, "Vietnamese,Chinese,Spanish": 0.5066666666666667, "Vietnamese,Chinese,Malay": 0.5333333333333333, "Vietnamese,Chinese,Indonesian": 0.54, "Vietnamese,Chinese,English": 0.56, "Vietnamese,Spanish,Malay": 0.5133333333333333, "Vietnamese,Spanish,Indonesian": 0.5466666666666666, "Vietnamese,Spanish,English": 0.54, "Vietnamese,Malay,Indonesian": 0.5866666666666667, "Vietnamese,Malay,English": 0.5666666666666667, "Vietnamese,Indonesian,English": 0.58, "Chinese,Spanish,Malay": 0.52, "Chinese,Spanish,Indonesian": 0.5466666666666666, "Chinese,Spanish,English": 0.56, "Chinese,Malay,Indonesian": 0.56, "Chinese,Malay,English": 0.5733333333333334, "Chinese,Indonesian,English": 0.6, "Spanish,Malay,Indonesian": 0.5533333333333333, "Spanish,Malay,English": 0.54, "Spanish,Indonesian,English": 0.58, "Malay,Indonesian,English": 0.6 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.32, "Filipino,Vietnamese,Chinese,Malay": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Chinese,English": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Malay": 0.31333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Spanish,English": 0.31333333333333335, "Filipino,Vietnamese,Malay,Indonesian": 0.36, "Filipino,Vietnamese,Malay,English": 0.36, "Filipino,Vietnamese,Indonesian,English": 0.36666666666666664, "Filipino,Chinese,Spanish,Malay": 0.34, "Filipino,Chinese,Spanish,Indonesian": 0.34, "Filipino,Chinese,Spanish,English": 0.3466666666666667, "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, "Filipino,Chinese,Malay,English": 0.38, "Filipino,Chinese,Indonesian,English": 0.3933333333333333, "Filipino,Spanish,Malay,Indonesian": 0.35333333333333333, "Filipino,Spanish,Malay,English": 0.3466666666666667, "Filipino,Spanish,Indonesian,English": 0.35333333333333333, "Filipino,Malay,Indonesian,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.46, "Vietnamese,Chinese,Spanish,Indonesian": 0.46, "Vietnamese,Chinese,Spanish,English": 0.47333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.49333333333333335, "Vietnamese,Chinese,Malay,English": 0.4866666666666667, "Vietnamese,Chinese,Indonesian,English": 0.5, "Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Spanish,Malay,English": 0.48, "Vietnamese,Spanish,Indonesian,English": 0.4866666666666667, "Vietnamese,Malay,Indonesian,English": 0.5266666666666666, "Chinese,Spanish,Malay,Indonesian": 0.4866666666666667, "Chinese,Spanish,Malay,English": 0.4866666666666667, "Chinese,Spanish,Indonesian,English": 0.5133333333333333, "Chinese,Malay,Indonesian,English": 0.5066666666666667, "Spanish,Malay,Indonesian,English": 0.5066666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.3, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Spanish,Malay,English": 0.3, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3, "Filipino,Vietnamese,Malay,Indonesian,English": 0.34, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.32, "Filipino,Chinese,Spanish,Indonesian,English": 0.32666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.3466666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.44, "Vietnamese,Chinese,Spanish,Malay,English": 0.44, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.44, "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian,English": 0.46, "Chinese,Spanish,Malay,Indonesian,English": 0.46 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.31333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.29333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.42 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.28 } }, "AC3_2": 0.5508852016235788, "AC3_3": 0.48961087795224245, "AC3_4": 0.44595899325724875, "AC3_5": 0.41145724675660356, "AC3_6": 0.3822192059245112, "AC3_7": 0.35573200987921605 }, "prompt_4": { "overall_acc": 0.4809523809523809, "language_acc": { "Filipino": 0.36, "Vietnamese": 0.4666666666666667, "Chinese": 0.47333333333333333, "Spanish": 0.4866666666666667, "Malay": 0.5133333333333333, "Indonesian": 0.5, "English": 0.5666666666666667 }, "consistency_score_2": 0.6301587301587301, "consistency_score_3": 0.48552380952380947, "consistency_score_4": 0.40590476190476193, "consistency_score_5": 0.353968253968254, "consistency_score_6": 0.31428571428571433, "consistency_score_7": 0.28, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5333333333333333, "Filipino,Chinese": 0.49333333333333335, "Filipino,Spanish": 0.5333333333333333, "Filipino,Malay": 0.5466666666666666, "Filipino,Indonesian": 0.5133333333333333, "Filipino,English": 0.52, "Vietnamese,Chinese": 0.62, "Vietnamese,Spanish": 0.6133333333333333, "Vietnamese,Malay": 0.6933333333333334, "Vietnamese,Indonesian": 0.68, "Vietnamese,English": 0.6466666666666666, "Chinese,Spanish": 0.5866666666666667, "Chinese,Malay": 0.68, "Chinese,Indonesian": 0.6866666666666666, "Chinese,English": 0.7133333333333334, "Spanish,Malay": 0.66, "Spanish,Indonesian": 0.64, "Spanish,English": 0.6866666666666666, "Malay,Indonesian": 0.7333333333333333, "Malay,English": 0.7333333333333333, "Indonesian,English": 0.72 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.38666666666666666, "Filipino,Vietnamese,Spanish": 0.4, "Filipino,Vietnamese,Malay": 0.43333333333333335, "Filipino,Vietnamese,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,English": 0.3933333333333333, "Filipino,Chinese,Spanish": 0.36666666666666664, "Filipino,Chinese,Malay": 0.4066666666666667, "Filipino,Chinese,Indonesian": 0.4, "Filipino,Chinese,English": 0.41333333333333333, "Filipino,Spanish,Malay": 0.44, "Filipino,Spanish,Indonesian": 0.3933333333333333, "Filipino,Spanish,English": 0.41333333333333333, "Filipino,Malay,Indonesian": 0.44666666666666666, "Filipino,Malay,English": 0.4533333333333333, "Filipino,Indonesian,English": 0.41333333333333333, "Vietnamese,Chinese,Spanish": 0.44666666666666666, "Vietnamese,Chinese,Malay": 0.54, "Vietnamese,Chinese,Indonesian": 0.5266666666666666, "Vietnamese,Chinese,English": 0.52, "Vietnamese,Spanish,Malay": 0.5266666666666666, "Vietnamese,Spanish,Indonesian": 0.5, "Vietnamese,Spanish,English": 0.52, "Vietnamese,Malay,Indonesian": 0.6, "Vietnamese,Malay,English": 0.5866666666666667, "Vietnamese,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay": 0.49333333333333335, "Chinese,Spanish,Indonesian": 0.4866666666666667, "Chinese,Spanish,English": 0.52, "Chinese,Malay,Indonesian": 0.5666666666666667, "Chinese,Malay,English": 0.5866666666666667, "Chinese,Indonesian,English": 0.5733333333333334, "Spanish,Malay,Indonesian": 0.54, "Spanish,Malay,English": 0.5733333333333334, "Spanish,Indonesian,English": 0.5466666666666666, "Malay,Indonesian,English": 0.6133333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.32, "Filipino,Vietnamese,Chinese,Malay": 0.36, "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Chinese,English": 0.3466666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.36, "Filipino,Vietnamese,Spanish,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,Spanish,English": 0.3333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.38, "Filipino,Vietnamese,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.36, "Filipino,Chinese,Spanish,Malay": 0.34, "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Chinese,Spanish,English": 0.34, "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, "Filipino,Chinese,Malay,English": 0.38666666666666666, "Filipino,Chinese,Indonesian,English": 0.36666666666666664, "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Spanish,Malay,English": 0.37333333333333335, "Filipino,Spanish,Indonesian,English": 0.3466666666666667, "Filipino,Malay,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.43333333333333335, "Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Spanish,English": 0.4266666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.5, "Vietnamese,Chinese,Malay,English": 0.4866666666666667, "Vietnamese,Chinese,Indonesian,English": 0.47333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.46, "Vietnamese,Spanish,Malay,English": 0.49333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.4533333333333333, "Vietnamese,Malay,Indonesian,English": 0.5266666666666666, "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Chinese,Spanish,Malay,English": 0.47333333333333333, "Chinese,Spanish,Indonesian,English": 0.46, "Chinese,Malay,Indonesian,English": 0.5133333333333333, "Spanish,Malay,Indonesian,English": 0.5 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.3, "Filipino,Vietnamese,Chinese,Spanish,English": 0.3, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.34, "Filipino,Vietnamese,Chinese,Malay,English": 0.34, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.32, "Filipino,Vietnamese,Spanish,Malay,English": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.30666666666666664, "Filipino,Vietnamese,Malay,Indonesian,English": 0.35333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32, "Filipino,Chinese,Spanish,Malay,English": 0.32666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.31333333333333335, "Filipino,Chinese,Malay,Indonesian,English": 0.35333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.42, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4, "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian,English": 0.44, "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.28 } }, "AC3_2": 0.545537414916888, "AC3_3": 0.4832272837685344, "AC3_4": 0.4402532318273013, "AC3_5": 0.40780372980811835, "AC3_6": 0.3801539777109644, "AC3_7": 0.3539424279885307 }, "prompt_5": { "overall_acc": 0.4638095238095237, "language_acc": { "Filipino": 0.36, "Vietnamese": 0.44, "Chinese": 0.4866666666666667, "Spanish": 0.44666666666666666, "Malay": 0.49333333333333335, "Indonesian": 0.47333333333333333, "English": 0.5466666666666666 }, "consistency_score_2": 0.6488888888888888, "consistency_score_3": 0.5167619047619048, "consistency_score_4": 0.44361904761904764, "consistency_score_5": 0.3958730158730158, "consistency_score_6": 0.3609523809523809, "consistency_score_7": 0.3333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.54, "Filipino,Chinese": 0.5466666666666666, "Filipino,Spanish": 0.56, "Filipino,Malay": 0.5533333333333333, "Filipino,Indonesian": 0.5333333333333333, "Filipino,English": 0.5533333333333333, "Vietnamese,Chinese": 0.64, "Vietnamese,Spanish": 0.6466666666666666, "Vietnamese,Malay": 0.72, "Vietnamese,Indonesian": 0.6733333333333333, "Vietnamese,English": 0.6933333333333334, "Chinese,Spanish": 0.6133333333333333, "Chinese,Malay": 0.7066666666666667, "Chinese,Indonesian": 0.7133333333333334, "Chinese,English": 0.7, "Spanish,Malay": 0.6333333333333333, "Spanish,Indonesian": 0.66, "Spanish,English": 0.66, "Malay,Indonesian": 0.7733333333333333, "Malay,English": 0.78, "Indonesian,English": 0.7266666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.44, "Filipino,Vietnamese,Spanish": 0.43333333333333335, "Filipino,Vietnamese,Malay": 0.46, "Filipino,Vietnamese,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,English": 0.43333333333333335, "Filipino,Chinese,Spanish": 0.43333333333333335, "Filipino,Chinese,Malay": 0.4666666666666667, "Filipino,Chinese,Indonesian": 0.44666666666666666, "Filipino,Chinese,English": 0.4533333333333333, "Filipino,Spanish,Malay": 0.44, "Filipino,Spanish,Indonesian": 0.43333333333333335, "Filipino,Spanish,English": 0.4533333333333333, "Filipino,Malay,Indonesian": 0.47333333333333333, "Filipino,Malay,English": 0.47333333333333333, "Filipino,Indonesian,English": 0.44, "Vietnamese,Chinese,Spanish": 0.5133333333333333, "Vietnamese,Chinese,Malay": 0.5733333333333334, "Vietnamese,Chinese,Indonesian": 0.5466666666666666, "Vietnamese,Chinese,English": 0.5533333333333333, "Vietnamese,Spanish,Malay": 0.5466666666666666, "Vietnamese,Spanish,Indonesian": 0.54, "Vietnamese,Spanish,English": 0.5466666666666666, "Vietnamese,Malay,Indonesian": 0.6266666666666667, "Vietnamese,Malay,English": 0.6266666666666667, "Vietnamese,Indonesian,English": 0.5666666666666667, "Chinese,Spanish,Malay": 0.5133333333333333, "Chinese,Spanish,Indonesian": 0.5266666666666666, "Chinese,Spanish,English": 0.5133333333333333, "Chinese,Malay,Indonesian": 0.6133333333333333, "Chinese,Malay,English": 0.6333333333333333, "Chinese,Indonesian,English": 0.5933333333333334, "Spanish,Malay,Indonesian": 0.56, "Spanish,Malay,English": 0.5733333333333334, "Spanish,Indonesian,English": 0.56, "Malay,Indonesian,English": 0.6533333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.3933333333333333, "Filipino,Vietnamese,Chinese,English": 0.4, "Filipino,Vietnamese,Spanish,Malay": 0.3933333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.38, "Filipino,Vietnamese,Spanish,English": 0.38666666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.38666666666666666, "Filipino,Chinese,Spanish,Malay": 0.4, "Filipino,Chinese,Spanish,Indonesian": 0.38666666666666666, "Filipino,Chinese,Spanish,English": 0.3933333333333333, "Filipino,Chinese,Malay,Indonesian": 0.42, "Filipino,Chinese,Malay,English": 0.43333333333333335, "Filipino,Chinese,Indonesian,English": 0.4, "Filipino,Spanish,Malay,Indonesian": 0.4066666666666667, "Filipino,Spanish,Malay,English": 0.4066666666666667, "Filipino,Spanish,Indonesian,English": 0.3933333333333333, "Filipino,Malay,Indonesian,English": 0.42, "Vietnamese,Chinese,Spanish,Malay": 0.47333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.5266666666666666, "Vietnamese,Chinese,Malay,English": 0.5266666666666666, "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, "Vietnamese,Spanish,Malay,Indonesian": 0.5, "Vietnamese,Spanish,Malay,English": 0.5066666666666667, "Vietnamese,Spanish,Indonesian,English": 0.48, "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay,Indonesian": 0.48, "Chinese,Spanish,Malay,English": 0.4866666666666667, "Chinese,Spanish,Indonesian,English": 0.47333333333333333, "Chinese,Malay,Indonesian,English": 0.5466666666666666, "Spanish,Malay,Indonesian,English": 0.52 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.36666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.36, "Filipino,Vietnamese,Chinese,Spanish,English": 0.36, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Malay,English": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.35333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.38, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.38, "Filipino,Chinese,Spanish,Malay,English": 0.38, "Filipino,Chinese,Spanish,Indonesian,English": 0.36666666666666664, "Filipino,Chinese,Malay,Indonesian,English": 0.38666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.38, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.44666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.43333333333333335, "Vietnamese,Chinese,Malay,Indonesian,English": 0.48, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.4533333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.3466666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.34, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.36, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3466666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.36, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.42 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 } }, "AC3_2": 0.5409567284345026, "AC3_3": 0.48885595880610466, "AC3_4": 0.45348966449762373, "AC3_5": 0.42715692475864026, "AC3_6": 0.40596722748844394, "AC3_7": 0.3878932695650393 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3717532467532468, "language_acc": { "Indonesian": 0.39204545454545453, "English": 0.35795454545454547, "Filipino": 0.30113636363636365, "Spanish": 0.3693181818181818, "Chinese": 0.45454545454545453, "Malay": 0.4034090909090909, "Vietnamese": 0.32386363636363635 }, "consistency_score_2": 0.5340909090909091, "consistency_score_3": 0.35259740259740263, "consistency_score_4": 0.2553571428571428, "consistency_score_5": 0.19453463203463206, "consistency_score_6": 0.15178571428571427, "consistency_score_7": 0.11931818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6136363636363636, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.5795454545454546, "Indonesian,Chinese": 0.5340909090909091, "Indonesian,Malay": 0.6988636363636364, "Indonesian,Vietnamese": 0.5568181818181818, "English,Filipino": 0.45454545454545453, "English,Spanish": 0.6193181818181818, "English,Chinese": 0.5113636363636364, "English,Malay": 0.5511363636363636, "English,Vietnamese": 0.5340909090909091, "Filipino,Spanish": 0.4943181818181818, "Filipino,Chinese": 0.3806818181818182, "Filipino,Malay": 0.4431818181818182, "Filipino,Vietnamese": 0.42045454545454547, "Spanish,Chinese": 0.5170454545454546, "Spanish,Malay": 0.6079545454545454, "Spanish,Vietnamese": 0.625, "Chinese,Malay": 0.5511363636363636, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.5625 }, "3_combine": { "Indonesian,English,Filipino": 0.3409090909090909, "Indonesian,English,Spanish": 0.4318181818181818, "Indonesian,English,Chinese": 0.3806818181818182, "Indonesian,English,Malay": 0.4659090909090909, "Indonesian,English,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish": 0.3465909090909091, "Indonesian,Filipino,Chinese": 0.26704545454545453, "Indonesian,Filipino,Malay": 0.3522727272727273, "Indonesian,Filipino,Vietnamese": 0.30113636363636365, "Indonesian,Spanish,Chinese": 0.36363636363636365, "Indonesian,Spanish,Malay": 0.48295454545454547, "Indonesian,Spanish,Vietnamese": 0.4147727272727273, "Indonesian,Chinese,Malay": 0.4318181818181818, "Indonesian,Chinese,Vietnamese": 0.32954545454545453, "Indonesian,Malay,Vietnamese": 0.4431818181818182, "English,Filipino,Spanish": 0.3409090909090909, "English,Filipino,Chinese": 0.23863636363636365, "English,Filipino,Malay": 0.2897727272727273, "English,Filipino,Vietnamese": 0.2727272727272727, "English,Spanish,Chinese": 0.3806818181818182, "English,Spanish,Malay": 0.4147727272727273, "English,Spanish,Vietnamese": 0.42045454545454547, "English,Chinese,Malay": 0.35795454545454547, "English,Chinese,Vietnamese": 0.3125, "English,Malay,Vietnamese": 0.35795454545454547, "Filipino,Spanish,Chinese": 0.2727272727272727, "Filipino,Spanish,Malay": 0.32954545454545453, "Filipino,Spanish,Vietnamese": 0.32386363636363635, "Filipino,Chinese,Malay": 0.2556818181818182, "Filipino,Chinese,Vietnamese": 0.2159090909090909, "Filipino,Malay,Vietnamese": 0.2840909090909091, "Spanish,Chinese,Malay": 0.39204545454545453, "Spanish,Chinese,Vietnamese": 0.35795454545454547, "Spanish,Malay,Vietnamese": 0.4375, "Chinese,Malay,Vietnamese": 0.3465909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2727272727272727, "Indonesian,English,Filipino,Chinese": 0.21022727272727273, "Indonesian,English,Filipino,Malay": 0.26704545454545453, "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Chinese": 0.30113636363636365, "Indonesian,English,Spanish,Malay": 0.36363636363636365, "Indonesian,English,Spanish,Vietnamese": 0.32386363636363635, "Indonesian,English,Chinese,Malay": 0.32386363636363635, "Indonesian,English,Chinese,Vietnamese": 0.25, "Indonesian,English,Malay,Vietnamese": 0.3125, "Indonesian,Filipino,Spanish,Chinese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Malay": 0.2897727272727273, "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Chinese,Malay": 0.2159090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Chinese,Malay": 0.32954545454545453, "Indonesian,Spanish,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.35795454545454547, "Indonesian,Chinese,Malay,Vietnamese": 0.2897727272727273, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.23863636363636365, "English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "English,Filipino,Chinese,Malay": 0.18181818181818182, "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "English,Filipino,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay": 0.2897727272727273, "English,Spanish,Chinese,Vietnamese": 0.2784090909090909, "English,Spanish,Malay,Vietnamese": 0.32386363636363635, "English,Chinese,Malay,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, "Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Malay": 0.22727272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "Indonesian,English,Filipino,Chinese,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, "English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 } }, "AC3_2": 0.4383756923595563, "AC3_3": 0.36192203130281486, "AC3_4": 0.3027532265335796, "AC3_5": 0.2554138407724643, "AC3_6": 0.21555924691342232, "AC3_7": 0.18065364383999674 }, "prompt_2": { "overall_acc": 0.37581168831168826, "language_acc": { "Indonesian": 0.4034090909090909, "English": 0.375, "Filipino": 0.29545454545454547, "Spanish": 0.32386363636363635, "Chinese": 0.42045454545454547, "Malay": 0.4375, "Vietnamese": 0.375 }, "consistency_score_2": 0.5284090909090909, "consistency_score_3": 0.341396103896104, "consistency_score_4": 0.24025974025974023, "consistency_score_5": 0.17775974025974026, "consistency_score_6": 0.13636363636363638, "consistency_score_7": 0.10795454545454546, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5909090909090909, "Indonesian,Filipino": 0.4375, "Indonesian,Spanish": 0.5568181818181818, "Indonesian,Chinese": 0.5681818181818182, "Indonesian,Malay": 0.6931818181818182, "Indonesian,Vietnamese": 0.5568181818181818, "English,Filipino": 0.42613636363636365, "English,Spanish": 0.5852272727272727, "English,Chinese": 0.5227272727272727, "English,Malay": 0.5625, "English,Vietnamese": 0.5170454545454546, "Filipino,Spanish": 0.44886363636363635, "Filipino,Chinese": 0.38636363636363635, "Filipino,Malay": 0.45454545454545453, "Filipino,Vietnamese": 0.4034090909090909, "Spanish,Chinese": 0.5170454545454546, "Spanish,Malay": 0.5795454545454546, "Spanish,Vietnamese": 0.5965909090909091, "Chinese,Malay": 0.5738636363636364, "Chinese,Vietnamese": 0.5568181818181818, "Malay,Vietnamese": 0.5625 }, "3_combine": { "Indonesian,English,Filipino": 0.29545454545454547, "Indonesian,English,Spanish": 0.3977272727272727, "Indonesian,English,Chinese": 0.375, "Indonesian,English,Malay": 0.44886363636363635, "Indonesian,English,Vietnamese": 0.3693181818181818, "Indonesian,Filipino,Spanish": 0.29545454545454547, "Indonesian,Filipino,Chinese": 0.26136363636363635, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Chinese": 0.375, "Indonesian,Spanish,Malay": 0.44886363636363635, "Indonesian,Spanish,Vietnamese": 0.39204545454545453, "Indonesian,Chinese,Malay": 0.45454545454545453, "Indonesian,Chinese,Vietnamese": 0.38636363636363635, "Indonesian,Malay,Vietnamese": 0.4375, "English,Filipino,Spanish": 0.2840909090909091, "English,Filipino,Chinese": 0.23295454545454544, "English,Filipino,Malay": 0.2897727272727273, "English,Filipino,Vietnamese": 0.24431818181818182, "English,Spanish,Chinese": 0.35795454545454547, "English,Spanish,Malay": 0.39204545454545453, "English,Spanish,Vietnamese": 0.3977272727272727, "English,Chinese,Malay": 0.36363636363636365, "English,Chinese,Vietnamese": 0.3352272727272727, "English,Malay,Vietnamese": 0.35795454545454547, "Filipino,Spanish,Chinese": 0.25, "Filipino,Spanish,Malay": 0.29545454545454547, "Filipino,Spanish,Vietnamese": 0.2784090909090909, "Filipino,Chinese,Malay": 0.2727272727272727, "Filipino,Chinese,Vietnamese": 0.23295454545454544, "Filipino,Malay,Vietnamese": 0.2727272727272727, "Spanish,Chinese,Malay": 0.38636363636363635, "Spanish,Chinese,Vietnamese": 0.3693181818181818, "Spanish,Malay,Vietnamese": 0.42045454545454547, "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2159090909090909, "Indonesian,English,Filipino,Chinese": 0.17613636363636365, "Indonesian,English,Filipino,Malay": 0.23295454545454544, "Indonesian,English,Filipino,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Chinese": 0.2784090909090909, "Indonesian,English,Spanish,Malay": 0.32386363636363635, "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,English,Chinese,Malay": 0.3181818181818182, "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,English,Malay,Vietnamese": 0.3068181818181818, "Indonesian,Filipino,Spanish,Chinese": 0.19318181818181818, "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Chinese,Malay": 0.2215909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Spanish,Chinese,Malay": 0.3465909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.3522727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.3181818181818182, "English,Filipino,Spanish,Chinese": 0.17045454545454544, "English,Filipino,Spanish,Malay": 0.19886363636363635, "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "English,Filipino,Chinese,Malay": 0.1875, "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "English,Filipino,Malay,Vietnamese": 0.19318181818181818, "English,Spanish,Chinese,Malay": 0.26704545454545453, "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "English,Spanish,Malay,Vietnamese": 0.30113636363636365, "English,Chinese,Malay,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, "Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.25, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.125, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 } }, "AC3_2": 0.43923412758321767, "AC3_3": 0.35777817130343365, "AC3_4": 0.29312321402425123, "AC3_5": 0.24135706664337872, "AC3_6": 0.20011525712848346, "AC3_7": 0.16772803535276004 }, "prompt_3": { "overall_acc": 0.38961038961038963, "language_acc": { "Indonesian": 0.45454545454545453, "English": 0.4034090909090909, "Filipino": 0.2840909090909091, "Spanish": 0.3352272727272727, "Chinese": 0.4318181818181818, "Malay": 0.42045454545454547, "Vietnamese": 0.3977272727272727 }, "consistency_score_2": 0.549512987012987, "consistency_score_3": 0.3704545454545455, "consistency_score_4": 0.26931818181818185, "consistency_score_5": 0.20454545454545456, "consistency_score_6": 0.1590909090909091, "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6363636363636364, "Indonesian,Filipino": 0.4659090909090909, "Indonesian,Spanish": 0.5738636363636364, "Indonesian,Chinese": 0.5397727272727273, "Indonesian,Malay": 0.7329545454545454, "Indonesian,Vietnamese": 0.5909090909090909, "English,Filipino": 0.4090909090909091, "English,Spanish": 0.5795454545454546, "English,Chinese": 0.5738636363636364, "English,Malay": 0.6022727272727273, "English,Vietnamese": 0.5795454545454546, "Filipino,Spanish": 0.4375, "Filipino,Chinese": 0.375, "Filipino,Malay": 0.4715909090909091, "Filipino,Vietnamese": 0.4318181818181818, "Spanish,Chinese": 0.5454545454545454, "Spanish,Malay": 0.5965909090909091, "Spanish,Vietnamese": 0.6704545454545454, "Chinese,Malay": 0.5397727272727273, "Chinese,Vietnamese": 0.5738636363636364, "Malay,Vietnamese": 0.6136363636363636 }, "3_combine": { "Indonesian,English,Filipino": 0.32954545454545453, "Indonesian,English,Spanish": 0.42613636363636365, "Indonesian,English,Chinese": 0.4034090909090909, "Indonesian,English,Malay": 0.5284090909090909, "Indonesian,English,Vietnamese": 0.42613636363636365, "Indonesian,Filipino,Spanish": 0.3181818181818182, "Indonesian,Filipino,Chinese": 0.26136363636363635, "Indonesian,Filipino,Malay": 0.38636363636363635, "Indonesian,Filipino,Vietnamese": 0.3125, "Indonesian,Spanish,Chinese": 0.35795454545454547, "Indonesian,Spanish,Malay": 0.48295454545454547, "Indonesian,Spanish,Vietnamese": 0.4431818181818182, "Indonesian,Chinese,Malay": 0.44886363636363635, "Indonesian,Chinese,Vietnamese": 0.3806818181818182, "Indonesian,Malay,Vietnamese": 0.5113636363636364, "English,Filipino,Spanish": 0.2840909090909091, "English,Filipino,Chinese": 0.24431818181818182, "English,Filipino,Malay": 0.30113636363636365, "English,Filipino,Vietnamese": 0.2784090909090909, "English,Spanish,Chinese": 0.39204545454545453, "English,Spanish,Malay": 0.4147727272727273, "English,Spanish,Vietnamese": 0.44886363636363635, "English,Chinese,Malay": 0.39204545454545453, "English,Chinese,Vietnamese": 0.4034090909090909, "English,Malay,Vietnamese": 0.42613636363636365, "Filipino,Spanish,Chinese": 0.23863636363636365, "Filipino,Spanish,Malay": 0.3068181818181818, "Filipino,Spanish,Vietnamese": 0.3125, "Filipino,Chinese,Malay": 0.2556818181818182, "Filipino,Chinese,Vietnamese": 0.25, "Filipino,Malay,Vietnamese": 0.32386363636363635, "Spanish,Chinese,Malay": 0.3806818181818182, "Spanish,Chinese,Vietnamese": 0.42045454545454547, "Spanish,Malay,Vietnamese": 0.4715909090909091, "Chinese,Malay,Vietnamese": 0.4034090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.24431818181818182, "Indonesian,English,Filipino,Chinese": 0.19886363636363635, "Indonesian,English,Filipino,Malay": 0.29545454545454547, "Indonesian,English,Filipino,Vietnamese": 0.23863636363636365, "Indonesian,English,Spanish,Chinese": 0.2897727272727273, "Indonesian,English,Spanish,Malay": 0.36363636363636365, "Indonesian,English,Spanish,Vietnamese": 0.3352272727272727, "Indonesian,English,Chinese,Malay": 0.35795454545454547, "Indonesian,English,Chinese,Vietnamese": 0.3068181818181818, "Indonesian,English,Malay,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.1875, "Indonesian,Filipino,Spanish,Malay": 0.2784090909090909, "Indonesian,Filipino,Spanish,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Chinese,Malay": 0.23863636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.30113636363636365, "Indonesian,Spanish,Chinese,Malay": 0.32386363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.3977272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.3352272727272727, "English,Filipino,Spanish,Chinese": 0.17613636363636365, "English,Filipino,Spanish,Malay": 0.2215909090909091, "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, "English,Filipino,Chinese,Malay": 0.19318181818181818, "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, "English,Filipino,Malay,Vietnamese": 0.23295454545454544, "English,Spanish,Chinese,Malay": 0.2897727272727273, "English,Spanish,Chinese,Vietnamese": 0.32386363636363635, "English,Spanish,Malay,Vietnamese": 0.3409090909090909, "English,Chinese,Malay,Vietnamese": 0.3068181818181818, "Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.3125 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1875, "Indonesian,English,Filipino,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.23295454545454544, "Indonesian,English,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3068181818181818, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, "English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1875, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, "AC3_2": 0.45594854578763583, "AC3_3": 0.3797910760247473, "AC3_4": 0.3184841765289693, "AC3_5": 0.26825633378495517, "AC3_6": 0.22592791819443553, "AC3_7": 0.18927444791274667 }, "prompt_4": { "overall_acc": 0.3766233766233766, "language_acc": { "Indonesian": 0.3806818181818182, "English": 0.4034090909090909, "Filipino": 0.3068181818181818, "Spanish": 0.3522727272727273, "Chinese": 0.42045454545454547, "Malay": 0.4090909090909091, "Vietnamese": 0.36363636363636365 }, "consistency_score_2": 0.5484307359307358, "consistency_score_3": 0.3693181818181818, "consistency_score_4": 0.26948051948051943, "consistency_score_5": 0.20589826839826836, "consistency_score_6": 0.16233766233766236, "consistency_score_7": 0.13068181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6477272727272727, "Indonesian,Filipino": 0.48295454545454547, "Indonesian,Spanish": 0.5965909090909091, "Indonesian,Chinese": 0.5568181818181818, "Indonesian,Malay": 0.6931818181818182, "Indonesian,Vietnamese": 0.5738636363636364, "English,Filipino": 0.48863636363636365, "English,Spanish": 0.6363636363636364, "English,Chinese": 0.6022727272727273, "English,Malay": 0.5909090909090909, "English,Vietnamese": 0.5625, "Filipino,Spanish": 0.48863636363636365, "Filipino,Chinese": 0.36363636363636365, "Filipino,Malay": 0.4602272727272727, "Filipino,Vietnamese": 0.39204545454545453, "Spanish,Chinese": 0.5113636363636364, "Spanish,Malay": 0.6079545454545454, "Spanish,Vietnamese": 0.5852272727272727, "Chinese,Malay": 0.5397727272727273, "Chinese,Vietnamese": 0.5340909090909091, "Malay,Vietnamese": 0.6022727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.3522727272727273, "Indonesian,English,Spanish": 0.4602272727272727, "Indonesian,English,Chinese": 0.4431818181818182, "Indonesian,English,Malay": 0.4943181818181818, "Indonesian,English,Vietnamese": 0.42613636363636365, "Indonesian,Filipino,Spanish": 0.3352272727272727, "Indonesian,Filipino,Chinese": 0.26704545454545453, "Indonesian,Filipino,Malay": 0.3522727272727273, "Indonesian,Filipino,Vietnamese": 0.30113636363636365, "Indonesian,Spanish,Chinese": 0.375, "Indonesian,Spanish,Malay": 0.4659090909090909, "Indonesian,Spanish,Vietnamese": 0.4034090909090909, "Indonesian,Chinese,Malay": 0.4375, "Indonesian,Chinese,Vietnamese": 0.375, "Indonesian,Malay,Vietnamese": 0.4659090909090909, "English,Filipino,Spanish": 0.3522727272727273, "English,Filipino,Chinese": 0.2897727272727273, "English,Filipino,Malay": 0.32954545454545453, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.42045454545454547, "English,Spanish,Malay": 0.4375, "English,Spanish,Vietnamese": 0.42045454545454547, "English,Chinese,Malay": 0.42045454545454547, "English,Chinese,Vietnamese": 0.3977272727272727, "English,Malay,Vietnamese": 0.3977272727272727, "Filipino,Spanish,Chinese": 0.2556818181818182, "Filipino,Spanish,Malay": 0.3352272727272727, "Filipino,Spanish,Vietnamese": 0.29545454545454547, "Filipino,Chinese,Malay": 0.26136363636363635, "Filipino,Chinese,Vietnamese": 0.22727272727272727, "Filipino,Malay,Vietnamese": 0.29545454545454547, "Spanish,Chinese,Malay": 0.375, "Spanish,Chinese,Vietnamese": 0.3693181818181818, "Spanish,Malay,Vietnamese": 0.42045454545454547, "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.26704545454545453, "Indonesian,English,Filipino,Chinese": 0.2215909090909091, "Indonesian,English,Filipino,Malay": 0.2784090909090909, "Indonesian,English,Filipino,Vietnamese": 0.25, "Indonesian,English,Spanish,Chinese": 0.3352272727272727, "Indonesian,English,Spanish,Malay": 0.35795454545454547, "Indonesian,English,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,English,Chinese,Malay": 0.3693181818181818, "Indonesian,English,Chinese,Vietnamese": 0.3125, "Indonesian,English,Malay,Vietnamese": 0.36363636363636365, "Indonesian,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Malay": 0.26704545454545453, "Indonesian,Filipino,Spanish,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Chinese,Malay": 0.2215909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Spanish,Chinese,Malay": 0.3068181818181818, "Indonesian,Spanish,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.3522727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.32954545454545453, "English,Filipino,Spanish,Chinese": 0.2215909090909091, "English,Filipino,Spanish,Malay": 0.2556818181818182, "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, "English,Filipino,Chinese,Malay": 0.2215909090909091, "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, "English,Filipino,Malay,Vietnamese": 0.2215909090909091, "English,Spanish,Chinese,Malay": 0.32386363636363635, "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, "English,Spanish,Malay,Vietnamese": 0.32386363636363635, "English,Chinese,Malay,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.1875, "Spanish,Chinese,Malay,Vietnamese": 0.2897727272727273 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.18181818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.2159090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,English,Filipino,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.2784090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.25, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.25, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 } }, "AC3_2": 0.4465724389196009, "AC3_3": 0.37293500835835375, "AC3_4": 0.3141682437678599, "AC3_5": 0.2662428142695661, "AC3_6": 0.2268815521406609, "AC3_7": 0.19403636359811519 }, "prompt_5": { "overall_acc": 0.38636363636363635, "language_acc": { "Indonesian": 0.42613636363636365, "English": 0.4147727272727273, "Filipino": 0.26136363636363635, "Spanish": 0.36363636363636365, "Chinese": 0.4375, "Malay": 0.42613636363636365, "Vietnamese": 0.375 }, "consistency_score_2": 0.5533008658008659, "consistency_score_3": 0.375487012987013, "consistency_score_4": 0.2788961038961038, "consistency_score_5": 0.21780303030303033, "consistency_score_6": 0.1728896103896104, "consistency_score_7": 0.13636363636363635, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6193181818181818, "Indonesian,Filipino": 0.4375, "Indonesian,Spanish": 0.5738636363636364, "Indonesian,Chinese": 0.5965909090909091, "Indonesian,Malay": 0.7045454545454546, "Indonesian,Vietnamese": 0.6079545454545454, "English,Filipino": 0.4147727272727273, "English,Spanish": 0.5681818181818182, "English,Chinese": 0.5965909090909091, "English,Malay": 0.5965909090909091, "English,Vietnamese": 0.6193181818181818, "Filipino,Spanish": 0.42613636363636365, "Filipino,Chinese": 0.38636363636363635, "Filipino,Malay": 0.4318181818181818, "Filipino,Vietnamese": 0.44886363636363635, "Spanish,Chinese": 0.5625, "Spanish,Malay": 0.5852272727272727, "Spanish,Vietnamese": 0.6306818181818182, "Chinese,Malay": 0.5909090909090909, "Chinese,Vietnamese": 0.5909090909090909, "Malay,Vietnamese": 0.6306818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.30113636363636365, "Indonesian,English,Spanish": 0.4090909090909091, "Indonesian,English,Chinese": 0.4375, "Indonesian,English,Malay": 0.48295454545454547, "Indonesian,English,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Spanish": 0.29545454545454547, "Indonesian,Filipino,Chinese": 0.2784090909090909, "Indonesian,Filipino,Malay": 0.3352272727272727, "Indonesian,Filipino,Vietnamese": 0.3068181818181818, "Indonesian,Spanish,Chinese": 0.4034090909090909, "Indonesian,Spanish,Malay": 0.44886363636363635, "Indonesian,Spanish,Vietnamese": 0.44886363636363635, "Indonesian,Chinese,Malay": 0.4772727272727273, "Indonesian,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,Malay,Vietnamese": 0.5, "English,Filipino,Spanish": 0.2727272727272727, "English,Filipino,Chinese": 0.2556818181818182, "English,Filipino,Malay": 0.2897727272727273, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.4090909090909091, "English,Spanish,Malay": 0.39204545454545453, "English,Spanish,Vietnamese": 0.44886363636363635, "English,Chinese,Malay": 0.4318181818181818, "English,Chinese,Vietnamese": 0.4375, "English,Malay,Vietnamese": 0.4602272727272727, "Filipino,Spanish,Chinese": 0.26136363636363635, "Filipino,Spanish,Malay": 0.29545454545454547, "Filipino,Spanish,Vietnamese": 0.3068181818181818, "Filipino,Chinese,Malay": 0.26136363636363635, "Filipino,Chinese,Vietnamese": 0.2784090909090909, "Filipino,Malay,Vietnamese": 0.32954545454545453, "Spanish,Chinese,Malay": 0.4090909090909091, "Spanish,Chinese,Vietnamese": 0.42045454545454547, "Spanish,Malay,Vietnamese": 0.45454545454545453, "Chinese,Malay,Vietnamese": 0.4318181818181818 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2215909090909091, "Indonesian,English,Filipino,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Malay": 0.24431818181818182, "Indonesian,English,Filipino,Vietnamese": 0.24431818181818182, "Indonesian,English,Spanish,Chinese": 0.32386363636363635, "Indonesian,English,Spanish,Malay": 0.32954545454545453, "Indonesian,English,Spanish,Vietnamese": 0.35795454545454547, "Indonesian,English,Chinese,Malay": 0.3693181818181818, "Indonesian,English,Chinese,Vietnamese": 0.3465909090909091, "Indonesian,English,Malay,Vietnamese": 0.39204545454545453, "Indonesian,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Malay": 0.2556818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.25, "Indonesian,Filipino,Chinese,Malay": 0.22727272727272727, "Indonesian,Filipino,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Chinese,Malay": 0.3409090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.3693181818181818, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.20454545454545456, "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, "English,Filipino,Chinese,Malay": 0.21022727272727273, "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, "English,Filipino,Malay,Vietnamese": 0.24431818181818182, "English,Spanish,Chinese,Malay": 0.3125, "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, "English,Spanish,Malay,Vietnamese": 0.35795454545454547, "English,Chinese,Malay,Vietnamese": 0.3465909090909091, "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.2215909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.1875, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,English,Filipino,Chinese,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.2727272727272727, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3125, "Indonesian,English,Chinese,Malay,Vietnamese": 0.3068181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 } }, "AC3_2": 0.45500353370547175, "AC3_3": 0.38084768412179404, "AC3_4": 0.3239495984250613, "AC3_5": 0.27856939294334765, "AC3_6": 0.23888375770896753, "AC3_7": 0.20158102762942087 } }, "sg_eval": { "prompt_1": { "accuracy": 0.4563106796116505 }, "prompt_2": { "accuracy": 0.5436893203883495 }, "prompt_3": { "accuracy": 0.5048543689320388 }, "prompt_4": { "accuracy": 0.4368932038834951 }, "prompt_5": { "accuracy": 0.4854368932038835 } }, "cn_eval": { "prompt_1": { "accuracy": 0.5238095238095238 }, "prompt_2": { "accuracy": 0.49523809523809526 }, "prompt_3": { "accuracy": 0.5142857142857142 }, "prompt_4": { "accuracy": 0.4857142857142857 }, "prompt_5": { "accuracy": 0.47619047619047616 } }, "us_eval": { "prompt_1": { "accuracy": 0.48598130841121495 }, "prompt_2": { "accuracy": 0.4672897196261682 }, "prompt_3": { "accuracy": 0.48598130841121495 }, "prompt_4": { "accuracy": 0.48598130841121495 }, "prompt_5": { "accuracy": 0.48598130841121495 } }, "ph_eval": { "prompt_1": { "accuracy": 0.33, "category_acc": { "brand": 0.3, "demographics": 0.4, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.1, "politics": 0.2, "culture": 0.4, "film": 0.4, "law": 0.3, "geography": 0.6 } }, "prompt_2": { "accuracy": 0.32, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.1, "politics": 0.2, "culture": 0.3, "film": 0.3, "law": 0.3, "geography": 0.8 } }, "prompt_3": { "accuracy": 0.38, "category_acc": { "brand": 0.5, "demographics": 0.2, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.2, "culture": 0.4, "film": 0.4, "law": 0.3, "geography": 0.9 } }, "prompt_4": { "accuracy": 0.31, "category_acc": { "brand": 0.2, "demographics": 0.0, "biology": 0.4, "history": 0.2, "literature": 0.1, "politics": 0.4, "culture": 0.3, "film": 0.4, "law": 0.3, "geography": 0.7 } }, "prompt_5": { "accuracy": 0.36, "category_acc": { "brand": 0.3, "demographics": 0.4, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.4, "culture": 0.3, "film": 0.4, "law": 0.3, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.1866125409947737 }, "prompt_2": { "bleu_score": 0.18638187736641829 }, "prompt_3": { "bleu_score": 0.18540130650550757 }, "prompt_4": { "bleu_score": 0.1880559410252317 }, "prompt_5": { "bleu_score": 0.15033083280606302 } }, "indommlu": { "prompt_1": { "accuracy": 0.4552373322651712, "category_acc": { "History": 0.43172690763052207, "Geography": 0.37755102040816324, "Lampungic": 0.4013605442176871, "Social science": 0.6978297161936561, "Balinese": 0.3078556263269639, "Makassarese": 0.34408602150537637, "Banjarese": 0.3958333333333333, "Chemistry": 0.2656934306569343, "Biology": 0.40828402366863903, "Science": 0.5366357069143447, "Christian religion": 0.5572139303482587, "Art": 0.5590682196339434, "Islam religion": 0.5689900426742532, "Hindu religion": 0.3933333333333333, "Madurese": 0.2983050847457627, "Sport": 0.4864864864864865, "Indonesian language": 0.5077833125778332, "Physics": 0.36363636363636365, "Minangkabau culture": 0.37185929648241206, "Dayak language": 0.26605504587155965, "Sociology": 0.46169354838709675, "Economy": 0.4180327868852459, "Sundanese": 0.4044943820224719, "Javanese": 0.3659274193548387, "Civic education": 0.5493562231759657 } }, "prompt_2": { "accuracy": 0.46404966953735227, "category_acc": { "History": 0.44377510040160645, "Geography": 0.3877551020408163, "Lampungic": 0.4013605442176871, "Social science": 0.6961602671118531, "Balinese": 0.3036093418259023, "Makassarese": 0.3602150537634409, "Banjarese": 0.4097222222222222, "Chemistry": 0.2671532846715328, "Biology": 0.4059171597633136, "Science": 0.5655314757481941, "Christian religion": 0.582089552238806, "Art": 0.5723793677204659, "Islam religion": 0.5846372688477952, "Hindu religion": 0.42, "Madurese": 0.3220338983050847, "Sport": 0.5067567567567568, "Indonesian language": 0.5121419676214197, "Physics": 0.3575757575757576, "Minangkabau culture": 0.4271356783919598, "Dayak language": 0.30275229357798167, "Sociology": 0.4939516129032258, "Economy": 0.42213114754098363, "Sundanese": 0.39671564390665515, "Javanese": 0.3790322580645161, "Civic education": 0.5579399141630901 } }, "prompt_3": { "accuracy": 0.4582415381534148, "category_acc": { "History": 0.41164658634538154, "Geography": 0.38979591836734695, "Lampungic": 0.4217687074829932, "Social science": 0.6844741235392321, "Balinese": 0.3354564755838641, "Makassarese": 0.40860215053763443, "Banjarese": 0.375, "Chemistry": 0.24525547445255474, "Biology": 0.42248520710059173, "Science": 0.5614035087719298, "Christian religion": 0.5671641791044776, "Art": 0.5773710482529119, "Islam religion": 0.5633001422475107, "Hindu religion": 0.46, "Madurese": 0.2677966101694915, "Sport": 0.47297297297297297, "Indonesian language": 0.5161892901618929, "Physics": 0.36363636363636365, "Minangkabau culture": 0.39195979899497485, "Dayak language": 0.3119266055045872, "Sociology": 0.49193548387096775, "Economy": 0.41598360655737704, "Sundanese": 0.3863439930855661, "Javanese": 0.35080645161290325, "Civic education": 0.5321888412017167 } }, "prompt_4": { "accuracy": 0.4557714133119701, "category_acc": { "History": 0.4397590361445783, "Geography": 0.37142857142857144, "Lampungic": 0.32653061224489793, "Social science": 0.679465776293823, "Balinese": 0.3099787685774947, "Makassarese": 0.3763440860215054, "Banjarese": 0.3541666666666667, "Chemistry": 0.272992700729927, "Biology": 0.40946745562130177, "Science": 0.5490196078431373, "Christian religion": 0.5771144278606966, "Art": 0.5490848585690515, "Islam religion": 0.5761024182076814, "Hindu religion": 0.43333333333333335, "Madurese": 0.3050847457627119, "Sport": 0.43243243243243246, "Indonesian language": 0.5158779576587795, "Physics": 0.3878787878787879, "Minangkabau culture": 0.3869346733668342, "Dayak language": 0.29357798165137616, "Sociology": 0.4737903225806452, "Economy": 0.430327868852459, "Sundanese": 0.38807260155574763, "Javanese": 0.3467741935483871, "Civic education": 0.5336194563662375 } }, "prompt_5": { "accuracy": 0.4355430936644636, "category_acc": { "History": 0.41566265060240964, "Geography": 0.35306122448979593, "Lampungic": 0.3673469387755102, "Social science": 0.6477462437395659, "Balinese": 0.2802547770700637, "Makassarese": 0.34408602150537637, "Banjarese": 0.3680555555555556, "Chemistry": 0.2671532846715328, "Biology": 0.3857988165680473, "Science": 0.5211558307533539, "Christian religion": 0.5422885572139303, "Art": 0.5291181364392679, "Islam religion": 0.5405405405405406, "Hindu religion": 0.41333333333333333, "Madurese": 0.3254237288135593, "Sport": 0.4527027027027027, "Indonesian language": 0.4953300124533001, "Physics": 0.3898989898989899, "Minangkabau culture": 0.34673366834170855, "Dayak language": 0.24770642201834864, "Sociology": 0.4435483870967742, "Economy": 0.4077868852459016, "Sundanese": 0.3500432152117545, "Javanese": 0.3336693548387097, "Civic education": 0.5321888412017167 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.3236011529924488 }, "prompt_2": { "bleu_score": 0.32440891006073647 }, "prompt_3": { "bleu_score": 0.3204429258830498 }, "prompt_4": { "bleu_score": 0.32704458117805263 }, "prompt_5": { "bleu_score": 0.31802407870582927 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.27250852620501337 }, "prompt_2": { "bleu_score": 0.26977172917246206 }, "prompt_3": { "bleu_score": 0.26954366575043015 }, "prompt_4": { "bleu_score": 0.2723245835557172 }, "prompt_5": { "bleu_score": 0.2629241654872022 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.18710585898067542 }, "prompt_2": { "bleu_score": 0.18518476878559328 }, "prompt_3": { "bleu_score": 0.1814602853117574 }, "prompt_4": { "bleu_score": 0.18619098922321092 }, "prompt_5": { "bleu_score": 0.18341704036900028 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.33312415897753844 }, "prompt_2": { "bleu_score": 0.33441784160936017 }, "prompt_3": { "bleu_score": 0.3288102573880272 }, "prompt_4": { "bleu_score": 0.33525485700633156 }, "prompt_5": { "bleu_score": 0.32441550434919486 } }, "mmlu": { "prompt_1": { "accuracy": 0.4515752625437573 }, "prompt_2": { "accuracy": 0.4690781796966161 }, "prompt_3": { "accuracy": 0.4690781796966161 }, "prompt_4": { "accuracy": 0.46557759626604434 }, "prompt_5": { "accuracy": 0.4469078179696616 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.42452627815516625, "category_acc": { "high_school_european_history": 0.5670731707317073, "business_ethics": 0.45454545454545453, "clinical_knowledge": 0.4356060606060606, "medical_genetics": 0.5050505050505051, "high_school_us_history": 0.5024630541871922, "high_school_physics": 0.25333333333333335, "high_school_world_history": 0.6059322033898306, "virology": 0.4121212121212121, "high_school_microeconomics": 0.3881856540084388, "econometrics": 0.26548672566371684, "college_computer_science": 0.29292929292929293, "high_school_biology": 0.44336569579288027, "abstract_algebra": 0.24242424242424243, "professional_accounting": 0.3487544483985765, "philosophy": 0.49032258064516127, "professional_medicine": 0.3284132841328413, "nutrition": 0.4131147540983607, "global_facts": 0.3434343434343434, "machine_learning": 0.3783783783783784, "security_studies": 0.4016393442622951, "public_relations": 0.5137614678899083, "professional_psychology": 0.40098199672667756, "prehistory": 0.47678018575851394, "anatomy": 0.4626865671641791, "human_sexuality": 0.5, "college_medicine": 0.36627906976744184, "high_school_government_and_politics": 0.609375, "college_chemistry": 0.3434343434343434, "logical_fallacies": 0.5370370370370371, "high_school_geography": 0.5532994923857868, "elementary_mathematics": 0.3103448275862069, "human_aging": 0.5045045045045045, "college_mathematics": 0.3333333333333333, "high_school_psychology": 0.5753676470588235, "formal_logic": 0.256, "high_school_statistics": 0.2651162790697674, "international_law": 0.575, "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.45454545454545453, "conceptual_physics": 0.34615384615384615, "miscellaneous": 0.6214833759590793, "high_school_chemistry": 0.31683168316831684, "marketing": 0.6738197424892703, "professional_law": 0.31833007175472927, "management": 0.6274509803921569, "college_physics": 0.21782178217821782, "jurisprudence": 0.48598130841121495, "world_religions": 0.5882352941176471, "sociology": 0.565, "us_foreign_policy": 0.6565656565656566, "high_school_macroeconomics": 0.38817480719794345, "computer_security": 0.5656565656565656, "moral_scenarios": 0.2516778523489933, "moral_disputes": 0.4753623188405797, "electrical_engineering": 0.4166666666666667, "astronomy": 0.47019867549668876, "college_biology": 0.46153846153846156 } }, "prompt_2": { "accuracy": 0.4466928852341795, "category_acc": { "high_school_european_history": 0.5548780487804879, "business_ethics": 0.45454545454545453, "clinical_knowledge": 0.45454545454545453, "medical_genetics": 0.494949494949495, "high_school_us_history": 0.5369458128078818, "high_school_physics": 0.28, "high_school_world_history": 0.597457627118644, "virology": 0.41818181818181815, "high_school_microeconomics": 0.4430379746835443, "econometrics": 0.2831858407079646, "college_computer_science": 0.36363636363636365, "high_school_biology": 0.4919093851132686, "abstract_algebra": 0.20202020202020202, "professional_accounting": 0.3665480427046263, "philosophy": 0.49032258064516127, "professional_medicine": 0.33948339483394835, "nutrition": 0.4721311475409836, "global_facts": 0.30303030303030304, "machine_learning": 0.3063063063063063, "security_studies": 0.4385245901639344, "public_relations": 0.5137614678899083, "professional_psychology": 0.4386252045826514, "prehistory": 0.5139318885448917, "anatomy": 0.5, "human_sexuality": 0.5076923076923077, "college_medicine": 0.4069767441860465, "high_school_government_and_politics": 0.5989583333333334, "college_chemistry": 0.3333333333333333, "logical_fallacies": 0.5802469135802469, "high_school_geography": 0.6091370558375635, "elementary_mathematics": 0.32625994694960214, "human_aging": 0.5180180180180181, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.6102941176470589, "formal_logic": 0.256, "high_school_statistics": 0.29767441860465116, "international_law": 0.625, "high_school_mathematics": 0.29739776951672864, "high_school_computer_science": 0.48484848484848486, "conceptual_physics": 0.3547008547008547, "miscellaneous": 0.649616368286445, "high_school_chemistry": 0.36633663366336633, "marketing": 0.7167381974248928, "professional_law": 0.3424657534246575, "management": 0.6078431372549019, "college_physics": 0.24752475247524752, "jurisprudence": 0.5327102803738317, "world_religions": 0.5764705882352941, "sociology": 0.615, "us_foreign_policy": 0.6767676767676768, "high_school_macroeconomics": 0.41645244215938304, "computer_security": 0.5555555555555556, "moral_scenarios": 0.25727069351230425, "moral_disputes": 0.4927536231884058, "electrical_engineering": 0.4583333333333333, "astronomy": 0.5231788079470199, "college_biology": 0.4755244755244755 } }, "prompt_3": { "accuracy": 0.4493385770468359, "category_acc": { "high_school_european_history": 0.5853658536585366, "business_ethics": 0.43434343434343436, "clinical_knowledge": 0.45075757575757575, "medical_genetics": 0.5252525252525253, "high_school_us_history": 0.5763546798029556, "high_school_physics": 0.24666666666666667, "high_school_world_history": 0.5932203389830508, "virology": 0.3878787878787879, "high_school_microeconomics": 0.4472573839662447, "econometrics": 0.2831858407079646, "college_computer_science": 0.37373737373737376, "high_school_biology": 0.5048543689320388, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.3309608540925267, "philosophy": 0.46774193548387094, "professional_medicine": 0.33210332103321033, "nutrition": 0.46229508196721314, "global_facts": 0.37373737373737376, "machine_learning": 0.3333333333333333, "security_studies": 0.45491803278688525, "public_relations": 0.5412844036697247, "professional_psychology": 0.43044189852700493, "prehistory": 0.5325077399380805, "anatomy": 0.5, "human_sexuality": 0.5153846153846153, "college_medicine": 0.38372093023255816, "high_school_government_and_politics": 0.6145833333333334, "college_chemistry": 0.3838383838383838, "logical_fallacies": 0.5740740740740741, "high_school_geography": 0.5786802030456852, "elementary_mathematics": 0.3129973474801061, "human_aging": 0.5, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.6158088235294118, "formal_logic": 0.28, "high_school_statistics": 0.27906976744186046, "international_law": 0.625, "high_school_mathematics": 0.30855018587360594, "high_school_computer_science": 0.47474747474747475, "conceptual_physics": 0.358974358974359, "miscellaneous": 0.6470588235294118, "high_school_chemistry": 0.32673267326732675, "marketing": 0.7467811158798283, "professional_law": 0.34833659491193736, "management": 0.6666666666666666, "college_physics": 0.2871287128712871, "jurisprudence": 0.5046728971962616, "world_religions": 0.5647058823529412, "sociology": 0.665, "us_foreign_policy": 0.6565656565656566, "high_school_macroeconomics": 0.4087403598971722, "computer_security": 0.5757575757575758, "moral_scenarios": 0.28523489932885904, "moral_disputes": 0.4811594202898551, "electrical_engineering": 0.4305555555555556, "astronomy": 0.5165562913907285, "college_biology": 0.46853146853146854 } }, "prompt_4": { "accuracy": 0.443618162316768, "category_acc": { "high_school_european_history": 0.5670731707317073, "business_ethics": 0.494949494949495, "clinical_knowledge": 0.45454545454545453, "medical_genetics": 0.48484848484848486, "high_school_us_history": 0.5763546798029556, "high_school_physics": 0.28, "high_school_world_history": 0.6101694915254238, "virology": 0.4, "high_school_microeconomics": 0.42616033755274263, "econometrics": 0.23008849557522124, "college_computer_science": 0.35353535353535354, "high_school_biology": 0.4919093851132686, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.3594306049822064, "philosophy": 0.4838709677419355, "professional_medicine": 0.3505535055350554, "nutrition": 0.4459016393442623, "global_facts": 0.36363636363636365, "machine_learning": 0.32432432432432434, "security_studies": 0.45491803278688525, "public_relations": 0.5045871559633027, "professional_psychology": 0.4402618657937807, "prehistory": 0.5170278637770898, "anatomy": 0.5149253731343284, "human_sexuality": 0.4846153846153846, "college_medicine": 0.36046511627906974, "high_school_government_and_politics": 0.6197916666666666, "college_chemistry": 0.30303030303030304, "logical_fallacies": 0.5740740740740741, "high_school_geography": 0.5939086294416244, "elementary_mathematics": 0.32360742705570295, "human_aging": 0.5180180180180181, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.6047794117647058, "formal_logic": 0.184, "high_school_statistics": 0.2837209302325581, "international_law": 0.6166666666666667, "high_school_mathematics": 0.2788104089219331, "high_school_computer_science": 0.48484848484848486, "conceptual_physics": 0.36752136752136755, "miscellaneous": 0.6406649616368286, "high_school_chemistry": 0.3465346534653465, "marketing": 0.703862660944206, "professional_law": 0.34637964774951074, "management": 0.6372549019607843, "college_physics": 0.2079207920792079, "jurisprudence": 0.514018691588785, "world_religions": 0.5588235294117647, "sociology": 0.635, "us_foreign_policy": 0.6767676767676768, "high_school_macroeconomics": 0.390745501285347, "computer_security": 0.5858585858585859, "moral_scenarios": 0.2505592841163311, "moral_disputes": 0.5101449275362319, "electrical_engineering": 0.4583333333333333, "astronomy": 0.4768211920529801, "college_biology": 0.46153846153846156 } }, "prompt_5": { "accuracy": 0.43575259206292455, "category_acc": { "high_school_european_history": 0.5792682926829268, "business_ethics": 0.48484848484848486, "clinical_knowledge": 0.4583333333333333, "medical_genetics": 0.46464646464646464, "high_school_us_history": 0.5615763546798029, "high_school_physics": 0.26666666666666666, "high_school_world_history": 0.6228813559322034, "virology": 0.41818181818181815, "high_school_microeconomics": 0.39662447257383965, "econometrics": 0.23893805309734514, "college_computer_science": 0.32323232323232326, "high_school_biology": 0.4854368932038835, "abstract_algebra": 0.24242424242424243, "professional_accounting": 0.3487544483985765, "philosophy": 0.4838709677419355, "professional_medicine": 0.3505535055350554, "nutrition": 0.4426229508196721, "global_facts": 0.32323232323232326, "machine_learning": 0.3063063063063063, "security_studies": 0.44672131147540983, "public_relations": 0.5321100917431193, "professional_psychology": 0.4238952536824877, "prehistory": 0.5046439628482973, "anatomy": 0.4701492537313433, "human_sexuality": 0.5, "college_medicine": 0.37790697674418605, "high_school_government_and_politics": 0.6041666666666666, "college_chemistry": 0.3434343434343434, "logical_fallacies": 0.5493827160493827, "high_school_geography": 0.5736040609137056, "elementary_mathematics": 0.3156498673740053, "human_aging": 0.509009009009009, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.5900735294117647, "formal_logic": 0.256, "high_school_statistics": 0.24651162790697675, "international_law": 0.5666666666666667, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.45454545454545453, "conceptual_physics": 0.3547008547008547, "miscellaneous": 0.6355498721227621, "high_school_chemistry": 0.3415841584158416, "marketing": 0.703862660944206, "professional_law": 0.33529028049575993, "management": 0.6274509803921569, "college_physics": 0.21782178217821782, "jurisprudence": 0.4953271028037383, "world_religions": 0.5882352941176471, "sociology": 0.605, "us_foreign_policy": 0.6565656565656566, "high_school_macroeconomics": 0.37789203084832906, "computer_security": 0.5757575757575758, "moral_scenarios": 0.2494407158836689, "moral_disputes": 0.48405797101449277, "electrical_engineering": 0.4791666666666667, "astronomy": 0.47019867549668876, "college_biology": 0.46853146853146854 } } }, "c_eval": { "prompt_1": { "accuracy": 0.437592867756315 }, "prompt_2": { "accuracy": 0.4309063893016345 }, "prompt_3": { "accuracy": 0.43536404160475484 }, "prompt_4": { "accuracy": 0.4175334323922734 }, "prompt_5": { "accuracy": 0.42570579494799404 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.43586550435865506, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4166666666666667, "computer_architecture": 0.34615384615384615, "college_programming": 0.2857142857142857, "college_physics": 0.4166666666666667, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.3333333333333333, "metrology_engineer": 0.4482758620689655, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.375, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.5, "middle_school_physics": 0.5416666666666666, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.36666666666666664, "business_administration": 0.4473684210526316, "marxism": 0.5833333333333334, "mao_zedong_thought": 0.5862068965517241, "education_science": 0.6176470588235294, "teacher_qualification": 0.6530612244897959, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.5, "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.18518518518518517, "law": 0.27586206896551724, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.5, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.5833333333333334, "high_school_history": 0.64, "middle_school_history": 0.5555555555555556, "civil_servant": 0.3076923076923077, "sports_science": 0.375, "plant_protection": 0.5185185185185185, "basic_medicine": 0.375, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.6078431372549019, "accountant": 0.37037037037037035, "fire_engineer": 0.4722222222222222, "environmental_impact_assessment_engineer": 0.5277777777777778, "tax_accountant": 0.42592592592592593, "physician": 0.5555555555555556 } }, "prompt_2": { "accuracy": 0.4414694894146949, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.375, "computer_architecture": 0.34615384615384615, "college_programming": 0.3333333333333333, "college_physics": 0.4583333333333333, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.375, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.25, "middle_school_biology": 0.5384615384615384, "middle_school_physics": 0.5416666666666666, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.38333333333333336, "business_administration": 0.42105263157894735, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.6551724137931034, "education_science": 0.6470588235294118, "teacher_qualification": 0.6326530612244898, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.6538461538461539, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.5357142857142857, "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.3333333333333333, "law": 0.3448275862068966, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.5263157894736842, "professional_tour_guide": 0.47058823529411764, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.5833333333333334, "high_school_history": 0.64, "middle_school_history": 0.6296296296296297, "civil_servant": 0.36538461538461536, "sports_science": 0.5416666666666666, "plant_protection": 0.5555555555555556, "basic_medicine": 0.5, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.5294117647058824, "accountant": 0.3333333333333333, "fire_engineer": 0.4166666666666667, "environmental_impact_assessment_engineer": 0.5, "tax_accountant": 0.42592592592592593, "physician": 0.5 } }, "prompt_3": { "accuracy": 0.4564134495641345, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.375, "computer_architecture": 0.4230769230769231, "college_programming": 0.47619047619047616, "college_physics": 0.4583333333333333, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.047619047619047616, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.08695652173913043, "high_school_physics": 0.5416666666666666, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.5, "middle_school_physics": 0.5416666666666666, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.35, "business_administration": 0.3684210526315789, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.6551724137931034, "education_science": 0.6176470588235294, "teacher_qualification": 0.6938775510204082, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.5, "middle_school_politics": 0.6538461538461539, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.5714285714285714, "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.4444444444444444, "law": 0.2413793103448276, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.5, "professional_tour_guide": 0.5, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.5833333333333334, "high_school_history": 0.64, "middle_school_history": 0.5925925925925926, "civil_servant": 0.40384615384615385, "sports_science": 0.4583333333333333, "plant_protection": 0.5925925925925926, "basic_medicine": 0.5, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.6078431372549019, "accountant": 0.3888888888888889, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.46296296296296297, "physician": 0.5555555555555556 } }, "prompt_4": { "accuracy": 0.4315068493150685, "category_acc": { "computer_network": 0.375, "operating_system": 0.375, "computer_architecture": 0.38461538461538464, "college_programming": 0.3333333333333333, "college_physics": 0.5, "college_chemistry": 0.41379310344827586, "advanced_mathematics": 0.125, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.375, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.25, "middle_school_biology": 0.5769230769230769, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.3333333333333333, "business_administration": 0.4473684210526316, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.6551724137931034, "education_science": 0.6176470588235294, "teacher_qualification": 0.5918367346938775, "high_school_politics": 0.6666666666666666, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.25925925925925924, "law": 0.27586206896551724, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.47368421052631576, "professional_tour_guide": 0.5294117647058824, "legal_professional": 0.35714285714285715, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.64, "middle_school_history": 0.6296296296296297, "civil_servant": 0.4423076923076923, "sports_science": 0.4166666666666667, "plant_protection": 0.4444444444444444, "basic_medicine": 0.2916666666666667, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.5686274509803921, "accountant": 0.3148148148148148, "fire_engineer": 0.5277777777777778, "environmental_impact_assessment_engineer": 0.5, "tax_accountant": 0.4074074074074074, "physician": 0.48148148148148145 } }, "prompt_5": { "accuracy": 0.43586550435865506, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.375, "computer_architecture": 0.34615384615384615, "college_programming": 0.30952380952380953, "college_physics": 0.5, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.3333333333333333, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.375, "high_school_chemistry": 0.5, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.5, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.36666666666666664, "business_administration": 0.42105263157894735, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.6551724137931034, "education_science": 0.5882352941176471, "teacher_qualification": 0.6530612244897959, "high_school_politics": 0.6666666666666666, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.625, "logic": 0.3333333333333333, "law": 0.2413793103448276, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.5, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.5, "high_school_history": 0.68, "middle_school_history": 0.5555555555555556, "civil_servant": 0.3269230769230769, "sports_science": 0.4583333333333333, "plant_protection": 0.4074074074074074, "basic_medicine": 0.2916666666666667, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.5882352941176471, "accountant": 0.35185185185185186, "fire_engineer": 0.4444444444444444, "environmental_impact_assessment_engineer": 0.5833333333333334, "tax_accountant": 0.42592592592592593, "physician": 0.46296296296296297 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.48028673835125446 }, "prompt_2": { "accuracy": 0.4659498207885305 }, "prompt_3": { "accuracy": 0.46236559139784944 }, "prompt_4": { "accuracy": 0.44802867383512546 }, "prompt_5": { "accuracy": 0.46236559139784944 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.43645311690554306, "category_acc": { "agronomy": 0.39644970414201186, "anatomy": 0.35135135135135137, "ancient_chinese": 0.21951219512195122, "arts": 0.60625, "astronomy": 0.296969696969697, "business_ethics": 0.5119617224880383, "chinese_civil_service_exam": 0.41875, "chinese_driving_rule": 0.6793893129770993, "chinese_food_culture": 0.41911764705882354, "chinese_foreign_policy": 0.4953271028037383, "chinese_history": 0.47678018575851394, "chinese_literature": 0.3480392156862745, "chinese_teacher_qualification": 0.5363128491620112, "clinical_knowledge": 0.35864978902953587, "college_actuarial_science": 0.20754716981132076, "college_education": 0.616822429906542, "college_engineering_hydrology": 0.41509433962264153, "college_law": 0.3888888888888889, "college_mathematics": 0.3333333333333333, "college_medical_statistics": 0.37735849056603776, "college_medicine": 0.3992673992673993, "computer_science": 0.45098039215686275, "computer_security": 0.45614035087719296, "conceptual_physics": 0.47619047619047616, "construction_project_management": 0.34532374100719426, "economics": 0.44025157232704404, "education": 0.50920245398773, "electrical_engineering": 0.42441860465116277, "elementary_chinese": 0.42063492063492064, "elementary_commonsense": 0.494949494949495, "elementary_information_and_technology": 0.6596638655462185, "elementary_mathematics": 0.2826086956521739, "ethnology": 0.43703703703703706, "food_science": 0.46153846153846156, "genetics": 0.3693181818181818, "global_facts": 0.4228187919463087, "high_school_biology": 0.33727810650887574, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.4661016949152542, "high_school_mathematics": 0.2865853658536585, "high_school_physics": 0.37272727272727274, "high_school_politics": 0.40559440559440557, "human_sexuality": 0.4603174603174603, "international_law": 0.3891891891891892, "journalism": 0.5174418604651163, "jurisprudence": 0.45012165450121655, "legal_and_moral_basis": 0.7757009345794392, "logical": 0.4065040650406504, "machine_learning": 0.38524590163934425, "management": 0.48095238095238096, "marketing": 0.5444444444444444, "marxist_theory": 0.5079365079365079, "modern_chinese": 0.33620689655172414, "nutrition": 0.4482758620689655, "philosophy": 0.5142857142857142, "professional_accounting": 0.45714285714285713, "professional_law": 0.3744075829383886, "professional_medicine": 0.3271276595744681, "professional_psychology": 0.5043103448275862, "public_relations": 0.4540229885057471, "security_study": 0.5111111111111111, "sociology": 0.4469026548672566, "sports_science": 0.44242424242424244, "traditional_chinese_medicine": 0.3837837837837838, "virology": 0.40236686390532544, "world_history": 0.45962732919254656, "world_religions": 0.45 } }, "prompt_2": { "accuracy": 0.4392160248661717, "category_acc": { "agronomy": 0.39644970414201186, "anatomy": 0.32432432432432434, "ancient_chinese": 0.2926829268292683, "arts": 0.65, "astronomy": 0.2909090909090909, "business_ethics": 0.4880382775119617, "chinese_civil_service_exam": 0.41875, "chinese_driving_rule": 0.7099236641221374, "chinese_food_culture": 0.4411764705882353, "chinese_foreign_policy": 0.48598130841121495, "chinese_history": 0.49226006191950467, "chinese_literature": 0.35784313725490197, "chinese_teacher_qualification": 0.553072625698324, "clinical_knowledge": 0.3670886075949367, "college_actuarial_science": 0.22641509433962265, "college_education": 0.6261682242990654, "college_engineering_hydrology": 0.4056603773584906, "college_law": 0.39814814814814814, "college_mathematics": 0.3238095238095238, "college_medical_statistics": 0.39622641509433965, "college_medicine": 0.4139194139194139, "computer_science": 0.44607843137254904, "computer_security": 0.4327485380116959, "conceptual_physics": 0.48299319727891155, "construction_project_management": 0.37410071942446044, "economics": 0.46540880503144655, "education": 0.50920245398773, "electrical_engineering": 0.4418604651162791, "elementary_chinese": 0.4166666666666667, "elementary_commonsense": 0.48484848484848486, "elementary_information_and_technology": 0.6764705882352942, "elementary_mathematics": 0.2782608695652174, "ethnology": 0.4222222222222222, "food_science": 0.45454545454545453, "genetics": 0.3409090909090909, "global_facts": 0.42953020134228187, "high_school_biology": 0.3431952662721893, "high_school_chemistry": 0.29545454545454547, "high_school_geography": 0.4491525423728814, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.38181818181818183, "high_school_politics": 0.3986013986013986, "human_sexuality": 0.47619047619047616, "international_law": 0.372972972972973, "journalism": 0.5116279069767442, "jurisprudence": 0.4257907542579075, "legal_and_moral_basis": 0.780373831775701, "logical": 0.42276422764227645, "machine_learning": 0.39344262295081966, "management": 0.5333333333333333, "marketing": 0.5277777777777778, "marxist_theory": 0.5132275132275133, "modern_chinese": 0.3706896551724138, "nutrition": 0.46206896551724136, "philosophy": 0.45714285714285713, "professional_accounting": 0.44, "professional_law": 0.41232227488151657, "professional_medicine": 0.300531914893617, "professional_psychology": 0.5129310344827587, "public_relations": 0.47126436781609193, "security_study": 0.5333333333333333, "sociology": 0.4469026548672566, "sports_science": 0.43636363636363634, "traditional_chinese_medicine": 0.3621621621621622, "virology": 0.40828402366863903, "world_history": 0.45962732919254656, "world_religions": 0.46875 } }, "prompt_3": { "accuracy": 0.4429286824382663, "category_acc": { "agronomy": 0.4260355029585799, "anatomy": 0.34459459459459457, "ancient_chinese": 0.23780487804878048, "arts": 0.63125, "astronomy": 0.3151515151515151, "business_ethics": 0.48325358851674644, "chinese_civil_service_exam": 0.4125, "chinese_driving_rule": 0.7175572519083969, "chinese_food_culture": 0.4338235294117647, "chinese_foreign_policy": 0.4766355140186916, "chinese_history": 0.48606811145510836, "chinese_literature": 0.3872549019607843, "chinese_teacher_qualification": 0.547486033519553, "clinical_knowledge": 0.3670886075949367, "college_actuarial_science": 0.25471698113207547, "college_education": 0.6074766355140186, "college_engineering_hydrology": 0.39622641509433965, "college_law": 0.37962962962962965, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.4056603773584906, "college_medicine": 0.4175824175824176, "computer_science": 0.46568627450980393, "computer_security": 0.4502923976608187, "conceptual_physics": 0.4897959183673469, "construction_project_management": 0.4028776978417266, "economics": 0.4528301886792453, "education": 0.5153374233128835, "electrical_engineering": 0.42441860465116277, "elementary_chinese": 0.42857142857142855, "elementary_commonsense": 0.4595959595959596, "elementary_information_and_technology": 0.680672268907563, "elementary_mathematics": 0.2956521739130435, "ethnology": 0.43703703703703706, "food_science": 0.4405594405594406, "genetics": 0.3806818181818182, "global_facts": 0.4228187919463087, "high_school_biology": 0.3431952662721893, "high_school_chemistry": 0.3106060606060606, "high_school_geography": 0.4830508474576271, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.4, "high_school_politics": 0.4195804195804196, "human_sexuality": 0.46825396825396826, "international_law": 0.3891891891891892, "journalism": 0.5290697674418605, "jurisprudence": 0.44038929440389296, "legal_and_moral_basis": 0.7850467289719626, "logical": 0.4146341463414634, "machine_learning": 0.39344262295081966, "management": 0.5333333333333333, "marketing": 0.5388888888888889, "marxist_theory": 0.5238095238095238, "modern_chinese": 0.3620689655172414, "nutrition": 0.496551724137931, "philosophy": 0.47619047619047616, "professional_accounting": 0.4342857142857143, "professional_law": 0.3933649289099526, "professional_medicine": 0.32180851063829785, "professional_psychology": 0.5043103448275862, "public_relations": 0.47701149425287354, "security_study": 0.5185185185185185, "sociology": 0.4247787610619469, "sports_science": 0.44242424242424244, "traditional_chinese_medicine": 0.3783783783783784, "virology": 0.4260355029585799, "world_history": 0.4472049689440994, "world_religions": 0.4625 } }, "prompt_4": { "accuracy": 0.42764634778103955, "category_acc": { "agronomy": 0.38461538461538464, "anatomy": 0.33783783783783783, "ancient_chinese": 0.25, "arts": 0.59375, "astronomy": 0.2727272727272727, "business_ethics": 0.4784688995215311, "chinese_civil_service_exam": 0.4125, "chinese_driving_rule": 0.6641221374045801, "chinese_food_culture": 0.47058823529411764, "chinese_foreign_policy": 0.48598130841121495, "chinese_history": 0.4551083591331269, "chinese_literature": 0.37254901960784315, "chinese_teacher_qualification": 0.547486033519553, "clinical_knowledge": 0.350210970464135, "college_actuarial_science": 0.22641509433962265, "college_education": 0.5887850467289719, "college_engineering_hydrology": 0.3867924528301887, "college_law": 0.3888888888888889, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.4056603773584906, "college_medicine": 0.3882783882783883, "computer_science": 0.45588235294117646, "computer_security": 0.4502923976608187, "conceptual_physics": 0.4421768707482993, "construction_project_management": 0.39568345323741005, "economics": 0.42138364779874216, "education": 0.5214723926380368, "electrical_engineering": 0.43023255813953487, "elementary_chinese": 0.42857142857142855, "elementary_commonsense": 0.45454545454545453, "elementary_information_and_technology": 0.6554621848739496, "elementary_mathematics": 0.2782608695652174, "ethnology": 0.4, "food_science": 0.45454545454545453, "genetics": 0.3522727272727273, "global_facts": 0.4228187919463087, "high_school_biology": 0.3254437869822485, "high_school_chemistry": 0.3484848484848485, "high_school_geography": 0.4576271186440678, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.34545454545454546, "high_school_politics": 0.4125874125874126, "human_sexuality": 0.42857142857142855, "international_law": 0.3783783783783784, "journalism": 0.4941860465116279, "jurisprudence": 0.43795620437956206, "legal_and_moral_basis": 0.7663551401869159, "logical": 0.43902439024390244, "machine_learning": 0.36885245901639346, "management": 0.5142857142857142, "marketing": 0.5333333333333333, "marxist_theory": 0.4603174603174603, "modern_chinese": 0.31896551724137934, "nutrition": 0.42758620689655175, "philosophy": 0.4666666666666667, "professional_accounting": 0.45714285714285713, "professional_law": 0.3459715639810427, "professional_medicine": 0.31382978723404253, "professional_psychology": 0.4827586206896552, "public_relations": 0.4425287356321839, "security_study": 0.48148148148148145, "sociology": 0.4336283185840708, "sports_science": 0.42424242424242425, "traditional_chinese_medicine": 0.372972972972973, "virology": 0.38461538461538464, "world_history": 0.484472049689441, "world_religions": 0.43125 } }, "prompt_5": { "accuracy": 0.43653945777931275, "category_acc": { "agronomy": 0.40828402366863903, "anatomy": 0.36486486486486486, "ancient_chinese": 0.27439024390243905, "arts": 0.59375, "astronomy": 0.296969696969697, "business_ethics": 0.48325358851674644, "chinese_civil_service_exam": 0.43125, "chinese_driving_rule": 0.6793893129770993, "chinese_food_culture": 0.45588235294117646, "chinese_foreign_policy": 0.4953271028037383, "chinese_history": 0.4674922600619195, "chinese_literature": 0.39215686274509803, "chinese_teacher_qualification": 0.5307262569832403, "clinical_knowledge": 0.35443037974683544, "college_actuarial_science": 0.24528301886792453, "college_education": 0.5887850467289719, "college_engineering_hydrology": 0.3867924528301887, "college_law": 0.4074074074074074, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.44339622641509435, "college_medicine": 0.3992673992673993, "computer_science": 0.46568627450980393, "computer_security": 0.4853801169590643, "conceptual_physics": 0.48299319727891155, "construction_project_management": 0.39568345323741005, "economics": 0.4528301886792453, "education": 0.5337423312883436, "electrical_engineering": 0.4127906976744186, "elementary_chinese": 0.4246031746031746, "elementary_commonsense": 0.4898989898989899, "elementary_information_and_technology": 0.6596638655462185, "elementary_mathematics": 0.28695652173913044, "ethnology": 0.45185185185185184, "food_science": 0.46853146853146854, "genetics": 0.36363636363636365, "global_facts": 0.4161073825503356, "high_school_biology": 0.33136094674556216, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.4576271186440678, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.34545454545454546, "high_school_politics": 0.42657342657342656, "human_sexuality": 0.4365079365079365, "international_law": 0.35135135135135137, "journalism": 0.5174418604651163, "jurisprudence": 0.4306569343065693, "legal_and_moral_basis": 0.7710280373831776, "logical": 0.42276422764227645, "machine_learning": 0.36885245901639346, "management": 0.5047619047619047, "marketing": 0.5333333333333333, "marxist_theory": 0.5132275132275133, "modern_chinese": 0.35344827586206895, "nutrition": 0.4413793103448276, "philosophy": 0.4380952380952381, "professional_accounting": 0.46285714285714286, "professional_law": 0.3696682464454976, "professional_medicine": 0.32978723404255317, "professional_psychology": 0.4956896551724138, "public_relations": 0.4540229885057471, "security_study": 0.5037037037037037, "sociology": 0.43805309734513276, "sports_science": 0.43636363636363634, "traditional_chinese_medicine": 0.35135135135135137, "virology": 0.39644970414201186, "world_history": 0.4472049689440994, "world_religions": 0.45625 } } }, "zbench": { "prompt_1": { "accuracy": 0.18181818181818182 }, "prompt_2": { "accuracy": 0.2727272727272727 }, "prompt_3": { "accuracy": 0.21212121212121213 }, "prompt_4": { "accuracy": 0.24242424242424243 }, "prompt_5": { "accuracy": 0.24242424242424243 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6295454545454545 }, "prompt_2": { "accuracy": 0.6613636363636364 }, "prompt_3": { "accuracy": 0.6772727272727272 }, "prompt_4": { "accuracy": 0.5727272727272728 }, "prompt_5": { "accuracy": 0.6068181818181818 } }, "ocnli": { "prompt_1": { "accuracy": 0.39796610169491525 }, "prompt_2": { "accuracy": 0.39084745762711864 }, "prompt_3": { "accuracy": 0.4040677966101695 }, "prompt_4": { "accuracy": 0.3827118644067797 }, "prompt_5": { "accuracy": 0.36135593220338985 } }, "c3": { "prompt_1": { "accuracy": 0.7774869109947644 }, "prompt_2": { "accuracy": 0.7729992520568437 }, "prompt_3": { "accuracy": 0.7778608825729244 }, "prompt_4": { "accuracy": 0.7741211667913238 }, "prompt_5": { "accuracy": 0.7696335078534031 } }, "dream": { "prompt_1": { "accuracy": 0.7143557079862812 }, "prompt_2": { "accuracy": 0.7261146496815286 }, "prompt_3": { "accuracy": 0.7275845173934345 }, "prompt_4": { "accuracy": 0.7035766780989711 }, "prompt_5": { "accuracy": 0.698187163155316 } }, "samsum": { "prompt_1": { "rouge1": 0.3313739994617144, "rouge2": 0.10653091519534055, "rougeL": 0.24154018848608264, "avg_rouge": 0.2264817010477125 }, "prompt_2": { "rouge1": 0.33083257013215717, "rouge2": 0.10462933943338155, "rougeL": 0.24150892295839743, "avg_rouge": 0.22565694417464535 }, "prompt_3": { "rouge1": 0.34835996028886995, "rouge2": 0.10982634542283559, "rougeL": 0.2575315647612755, "avg_rouge": 0.23857262349099365 }, "prompt_4": { "rouge1": 0.3349310528982967, "rouge2": 0.10951537129251994, "rougeL": 0.24659014274854002, "avg_rouge": 0.2303455223131189 }, "prompt_5": { "rouge1": 0.3428935036440538, "rouge2": 0.11041972210880038, "rougeL": 0.25412988565195127, "avg_rouge": 0.23581437046826847 } }, "dialogsum": { "prompt_1": { "rouge1": 0.21956463999285744, "rouge2": 0.05747048037363373, "rougeL": 0.1596940858570499, "avg_rouge": 0.1455764020745137 }, "prompt_2": { "rouge1": 0.2174604871339299, "rouge2": 0.05475645193832106, "rougeL": 0.15851239276061685, "avg_rouge": 0.14357644394428928 }, "prompt_3": { "rouge1": 0.217054414110199, "rouge2": 0.055400616357914836, "rougeL": 0.15783238159600757, "avg_rouge": 0.1434291373547071 }, "prompt_4": { "rouge1": 0.2188521463516572, "rouge2": 0.05698676971087247, "rougeL": 0.1595901861437309, "avg_rouge": 0.14514303406875353 }, "prompt_5": { "rouge1": 0.2217691129634524, "rouge2": 0.05864590992084784, "rougeL": 0.16271839979241123, "avg_rouge": 0.14771114089223714 } }, "sst2": { "prompt_1": { "accuracy": 0.9139908256880734 }, "prompt_2": { "accuracy": 0.9013761467889908 }, "prompt_3": { "accuracy": 0.9025229357798165 }, "prompt_4": { "accuracy": 0.9185779816513762 }, "prompt_5": { "accuracy": 0.9036697247706422 } }, "cola": { "prompt_1": { "accuracy": 0.5589645254074784 }, "prompt_2": { "accuracy": 0.6049856184084372 }, "prompt_3": { "accuracy": 0.6519654841802492 }, "prompt_4": { "accuracy": 0.6097794822627037 }, "prompt_5": { "accuracy": 0.6538830297219559 } }, "qqp": { "prompt_1": { "accuracy": 0.5695 }, "prompt_2": { "accuracy": 0.7025 }, "prompt_3": { "accuracy": 0.603 }, "prompt_4": { "accuracy": 0.623 }, "prompt_5": { "accuracy": 0.71 } }, "mnli": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.5 } }, "qnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 1.0 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "sailor_7b_chat": { "model_size": "7B", "model_link": "https://huggingface.co/sail/Sailor-7B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9147058823529411, "language_acc": { "Spanish": 0.9042016806722689, "English": 0.9512605042016806, "Chinese": 0.9042016806722689, "Vietnamese": 0.8991596638655462 }, "consistency_score_2": 0.8918767507002802, "consistency_score_3": 0.8445378151260504, "consistency_score_4": 0.8100840336134454, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.915126050420168, "Spanish,Chinese": 0.8798319327731092, "Spanish,Vietnamese": 0.8764705882352941, "English,Chinese": 0.8974789915966387, "English,Vietnamese": 0.9193277310924369, "Chinese,Vietnamese": 0.8630252100840337 }, "3_combine": { "Spanish,English,Chinese": 0.8529411764705882, "Spanish,English,Vietnamese": 0.8596638655462185, "Spanish,Chinese,Vietnamese": 0.8201680672268907, "English,Chinese,Vietnamese": 0.8453781512605042 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8100840336134454 } }, "AC3_2": 0.903147074734647, "AC3_3": 0.8782225093980909, "AC3_4": 0.8592218957730685 }, "prompt_2": { "overall_acc": 0.9201680672268907, "language_acc": { "Spanish": 0.9100840336134454, "English": 0.9495798319327731, "Chinese": 0.9084033613445378, "Vietnamese": 0.9126050420168067 }, "consistency_score_2": 0.8988795518207283, "consistency_score_3": 0.8535714285714286, "consistency_score_4": 0.819327731092437, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9159663865546218, "Spanish,Chinese": 0.8798319327731092, "Spanish,Vietnamese": 0.8840336134453781, "English,Chinese": 0.9092436974789916, "English,Vietnamese": 0.9218487394957983, "Chinese,Vietnamese": 0.8823529411764706 }, "3_combine": { "Spanish,English,Chinese": 0.8588235294117647, "Spanish,English,Vietnamese": 0.865546218487395, "Spanish,Chinese,Vietnamese": 0.8294117647058824, "English,Chinese,Vietnamese": 0.8605042016806723 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.819327731092437 } }, "AC3_2": 0.9093992385490498, "AC3_3": 0.8856195326144926, "AC3_4": 0.8668249908160796 }, "prompt_3": { "overall_acc": 0.9262605042016806, "language_acc": { "Spanish": 0.9218487394957983, "English": 0.9478991596638655, "Chinese": 0.9176470588235294, "Vietnamese": 0.9176470588235294 }, "consistency_score_2": 0.9099439775910364, "consistency_score_3": 0.8682773109243698, "consistency_score_4": 0.8369747899159664, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9310924369747899, "Spanish,Chinese": 0.8966386554621849, "Spanish,Vietnamese": 0.9, "English,Chinese": 0.9134453781512605, "English,Vietnamese": 0.9260504201680673, "Chinese,Vietnamese": 0.892436974789916 }, "3_combine": { "Spanish,English,Chinese": 0.8739495798319328, "Spanish,English,Vietnamese": 0.8815126050420168, "Spanish,Chinese,Vietnamese": 0.8487394957983193, "English,Chinese,Vietnamese": 0.8689075630252101 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8369747899159664 } }, "AC3_2": 0.9180297464582713, "AC3_3": 0.896332161941517, "AC3_4": 0.8793570471893741 }, "prompt_4": { "overall_acc": 0.9159663865546218, "language_acc": { "Spanish": 0.9092436974789916, "English": 0.9504201680672268, "Chinese": 0.9033613445378151, "Vietnamese": 0.9008403361344538 }, "consistency_score_2": 0.892296918767507, "consistency_score_3": 0.8453781512605042, "consistency_score_4": 0.8117647058823529, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.915126050420168, "Spanish,Chinese": 0.880672268907563, "Spanish,Vietnamese": 0.8764705882352941, "English,Chinese": 0.9, "English,Vietnamese": 0.9142857142857143, "Chinese,Vietnamese": 0.8672268907563025 }, "3_combine": { "Spanish,English,Chinese": 0.853781512605042, "Spanish,English,Vietnamese": 0.8579831932773109, "Spanish,Chinese,Vietnamese": 0.8226890756302521, "English,Chinese,Vietnamese": 0.8470588235294118 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8117647058823529 } }, "AC3_2": 0.9039767405184379, "AC3_3": 0.879257809944307, "AC3_4": 0.8607232775880852 }, "prompt_5": { "overall_acc": 0.9197478991596638, "language_acc": { "Spanish": 0.9084033613445378, "English": 0.9394957983193277, "Chinese": 0.9159663865546218, "Vietnamese": 0.915126050420168 }, "consistency_score_2": 0.900140056022409, "consistency_score_3": 0.8546218487394959, "consistency_score_4": 0.8210084033613445, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9126050420168067, "Spanish,Chinese": 0.8831932773109243, "Spanish,Vietnamese": 0.8907563025210085, "English,Chinese": 0.9109243697478991, "English,Vietnamese": 0.9176470588235294, "Chinese,Vietnamese": 0.8857142857142857 }, "3_combine": { "Spanish,English,Chinese": 0.8579831932773109, "Spanish,English,Vietnamese": 0.8638655462184874, "Spanish,Chinese,Vietnamese": 0.8352941176470589, "English,Chinese,Vietnamese": 0.8613445378151261 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8210084033613445 } }, "AC3_2": 0.9098383480951743, "AC3_3": 0.8859896882715244, "AC3_4": 0.8675777913853445 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.5161904761904762, "language_acc": { "Filipino": 0.3466666666666667, "Vietnamese": 0.5266666666666666, "Chinese": 0.5666666666666667, "Spanish": 0.52, "Malay": 0.5, "Indonesian": 0.5266666666666666, "English": 0.6266666666666667 }, "consistency_score_2": 0.5923809523809523, "consistency_score_3": 0.43085714285714294, "consistency_score_4": 0.336, "consistency_score_5": 0.27142857142857146, "consistency_score_6": 0.22380952380952382, "consistency_score_7": 0.18666666666666668, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.42, "Filipino,Chinese": 0.4266666666666667, "Filipino,Spanish": 0.43333333333333335, "Filipino,Malay": 0.4066666666666667, "Filipino,Indonesian": 0.4066666666666667, "Filipino,English": 0.43333333333333335, "Vietnamese,Chinese": 0.6666666666666666, "Vietnamese,Spanish": 0.6533333333333333, "Vietnamese,Malay": 0.5866666666666667, "Vietnamese,Indonesian": 0.68, "Vietnamese,English": 0.6866666666666666, "Chinese,Spanish": 0.6466666666666666, "Chinese,Malay": 0.5733333333333334, "Chinese,Indonesian": 0.6266666666666667, "Chinese,English": 0.66, "Spanish,Malay": 0.64, "Spanish,Indonesian": 0.6733333333333333, "Spanish,English": 0.7, "Malay,Indonesian": 0.7066666666666667, "Malay,English": 0.6933333333333334, "Indonesian,English": 0.72 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.30666666666666664, "Filipino,Vietnamese,Spanish": 0.30666666666666664, "Filipino,Vietnamese,Malay": 0.2866666666666667, "Filipino,Vietnamese,Indonesian": 0.32, "Filipino,Vietnamese,English": 0.32666666666666666, "Filipino,Chinese,Spanish": 0.3, "Filipino,Chinese,Malay": 0.26666666666666666, "Filipino,Chinese,Indonesian": 0.2866666666666667, "Filipino,Chinese,English": 0.32, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.3333333333333333, "Filipino,Malay,Indonesian": 0.31333333333333335, "Filipino,Malay,English": 0.3, "Filipino,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish": 0.5066666666666667, "Vietnamese,Chinese,Malay": 0.4533333333333333, "Vietnamese,Chinese,Indonesian": 0.5066666666666667, "Vietnamese,Chinese,English": 0.5333333333333333, "Vietnamese,Spanish,Malay": 0.48, "Vietnamese,Spanish,Indonesian": 0.5466666666666666, "Vietnamese,Spanish,English": 0.5466666666666666, "Vietnamese,Malay,Indonesian": 0.52, "Vietnamese,Malay,English": 0.49333333333333335, "Vietnamese,Indonesian,English": 0.5666666666666667, "Chinese,Spanish,Malay": 0.48, "Chinese,Spanish,Indonesian": 0.5133333333333333, "Chinese,Spanish,English": 0.54, "Chinese,Malay,Indonesian": 0.49333333333333335, "Chinese,Malay,English": 0.49333333333333335, "Chinese,Indonesian,English": 0.5266666666666666, "Spanish,Malay,Indonesian": 0.5466666666666666, "Spanish,Malay,English": 0.54, "Spanish,Indonesian,English": 0.5733333333333334, "Malay,Indonesian,English": 0.6 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.24, "Filipino,Vietnamese,Chinese,Malay": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Spanish,English": 0.2733333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Chinese,Spanish,English": 0.2733333333333333, "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Chinese,Malay,English": 0.23333333333333334, "Filipino,Chinese,Indonesian,English": 0.26, "Filipino,Spanish,Malay,Indonesian": 0.26, "Filipino,Spanish,Malay,English": 0.26, "Filipino,Spanish,Indonesian,English": 0.2866666666666667, "Filipino,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay": 0.4, "Vietnamese,Chinese,Spanish,Indonesian": 0.44666666666666666, "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Chinese,Malay,English": 0.4, "Vietnamese,Chinese,Indonesian,English": 0.44666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.44666666666666666, "Vietnamese,Spanish,Malay,English": 0.43333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.4866666666666667, "Vietnamese,Malay,Indonesian,English": 0.46, "Chinese,Spanish,Malay,Indonesian": 0.43333333333333335, "Chinese,Spanish,Malay,English": 0.44, "Chinese,Spanish,Indonesian,English": 0.4666666666666667, "Chinese,Malay,Indonesian,English": 0.44666666666666666, "Spanish,Malay,Indonesian,English": 0.48 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.22, "Filipino,Vietnamese,Chinese,Spanish,English": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.2, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.22, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.24, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,English": 0.22, "Filipino,Chinese,Spanish,Indonesian,English": 0.24, "Filipino,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Malay,English": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.41333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.37333333333333335, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4066666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.35333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 } }, "AC3_2": 0.5516674848135973, "AC3_3": 0.4696793471518771, "AC3_4": 0.40704514970637046, "AC3_5": 0.35577819998937693, "AC3_6": 0.3122387693394321, "AC3_7": 0.27418247511907523 }, "prompt_2": { "overall_acc": 0.5342857142857144, "language_acc": { "Filipino": 0.36, "Vietnamese": 0.54, "Chinese": 0.5733333333333334, "Spanish": 0.5266666666666666, "Malay": 0.5466666666666666, "Indonesian": 0.5466666666666666, "English": 0.6466666666666666 }, "consistency_score_2": 0.5955555555555556, "consistency_score_3": 0.4342857142857143, "consistency_score_4": 0.34247619047619055, "consistency_score_5": 0.2812698412698412, "consistency_score_6": 0.2361904761904762, "consistency_score_7": 0.2, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4, "Filipino,Chinese": 0.44, "Filipino,Spanish": 0.4066666666666667, "Filipino,Malay": 0.3933333333333333, "Filipino,Indonesian": 0.4266666666666667, "Filipino,English": 0.44666666666666666, "Vietnamese,Chinese": 0.6333333333333333, "Vietnamese,Spanish": 0.6866666666666666, "Vietnamese,Malay": 0.6266666666666667, "Vietnamese,Indonesian": 0.6733333333333333, "Vietnamese,English": 0.7066666666666667, "Chinese,Spanish": 0.62, "Chinese,Malay": 0.6533333333333333, "Chinese,Indonesian": 0.6066666666666667, "Chinese,English": 0.6733333333333333, "Spanish,Malay": 0.6733333333333333, "Spanish,Indonesian": 0.68, "Spanish,English": 0.68, "Malay,Indonesian": 0.6866666666666666, "Malay,English": 0.7066666666666667, "Indonesian,English": 0.6866666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2866666666666667, "Filipino,Vietnamese,Spanish": 0.3, "Filipino,Vietnamese,Malay": 0.26666666666666666, "Filipino,Vietnamese,Indonesian": 0.3, "Filipino,Vietnamese,English": 0.31333333333333335, "Filipino,Chinese,Spanish": 0.28, "Filipino,Chinese,Malay": 0.31333333333333335, "Filipino,Chinese,Indonesian": 0.29333333333333333, "Filipino,Chinese,English": 0.32666666666666666, "Filipino,Spanish,Malay": 0.26666666666666666, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.30666666666666664, "Filipino,Malay,Indonesian": 0.29333333333333333, "Filipino,Malay,English": 0.32, "Filipino,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish": 0.5133333333333333, "Vietnamese,Chinese,Malay": 0.5133333333333333, "Vietnamese,Chinese,Indonesian": 0.49333333333333335, "Vietnamese,Chinese,English": 0.5333333333333333, "Vietnamese,Spanish,Malay": 0.5333333333333333, "Vietnamese,Spanish,Indonesian": 0.5533333333333333, "Vietnamese,Spanish,English": 0.5733333333333334, "Vietnamese,Malay,Indonesian": 0.54, "Vietnamese,Malay,English": 0.5533333333333333, "Vietnamese,Indonesian,English": 0.5666666666666667, "Chinese,Spanish,Malay": 0.52, "Chinese,Spanish,Indonesian": 0.5, "Chinese,Spanish,English": 0.5133333333333333, "Chinese,Malay,Indonesian": 0.5133333333333333, "Chinese,Malay,English": 0.5466666666666666, "Chinese,Indonesian,English": 0.5266666666666666, "Spanish,Malay,Indonesian": 0.5533333333333333, "Spanish,Malay,English": 0.54, "Spanish,Indonesian,English": 0.5466666666666666, "Malay,Indonesian,English": 0.5666666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Chinese,English": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Spanish,English": 0.2733333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Chinese,Spanish,English": 0.24666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, "Filipino,Chinese,Malay,English": 0.28, "Filipino,Chinese,Indonesian,English": 0.2733333333333333, "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Spanish,Malay,English": 0.24666666666666667, "Filipino,Spanish,Indonesian,English": 0.26666666666666666, "Filipino,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay": 0.4666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.44, "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.4533333333333333, "Vietnamese,Chinese,Malay,English": 0.4666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Spanish,Malay,English": 0.48, "Vietnamese,Spanish,Indonesian,English": 0.4866666666666667, "Vietnamese,Malay,Indonesian,English": 0.4866666666666667, "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Chinese,Spanish,Malay,English": 0.46, "Chinese,Spanish,Indonesian,English": 0.44666666666666666, "Chinese,Malay,Indonesian,English": 0.46, "Spanish,Malay,Indonesian,English": 0.47333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.22, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24, "Filipino,Vietnamese,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22, "Filipino,Chinese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Malay,Indonesian,English": 0.24, "Filipino,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.43333333333333335, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.41333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.42, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4266666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.4066666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2 } }, "AC3_2": 0.5632593424620886, "AC3_3": 0.4791234723483417, "AC3_4": 0.4173998323598293, "AC3_5": 0.3685302785512646, "AC3_6": 0.32757195828346014, "AC3_7": 0.2910505836179503 }, "prompt_3": { "overall_acc": 0.5361904761904762, "language_acc": { "Filipino": 0.36666666666666664, "Vietnamese": 0.52, "Chinese": 0.5666666666666667, "Spanish": 0.5533333333333333, "Malay": 0.5533333333333333, "Indonesian": 0.5333333333333333, "English": 0.66 }, "consistency_score_2": 0.5984126984126984, "consistency_score_3": 0.43866666666666665, "consistency_score_4": 0.34514285714285725, "consistency_score_5": 0.280952380952381, "consistency_score_6": 0.2323809523809524, "consistency_score_7": 0.19333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4066666666666667, "Filipino,Chinese": 0.44, "Filipino,Spanish": 0.46, "Filipino,Malay": 0.42, "Filipino,Indonesian": 0.4066666666666667, "Filipino,English": 0.4266666666666667, "Vietnamese,Chinese": 0.68, "Vietnamese,Spanish": 0.6666666666666666, "Vietnamese,Malay": 0.68, "Vietnamese,Indonesian": 0.6666666666666666, "Vietnamese,English": 0.6733333333333333, "Chinese,Spanish": 0.66, "Chinese,Malay": 0.6133333333333333, "Chinese,Indonesian": 0.6533333333333333, "Chinese,English": 0.66, "Spanish,Malay": 0.6866666666666666, "Spanish,Indonesian": 0.6666666666666666, "Spanish,English": 0.6866666666666666, "Malay,Indonesian": 0.7, "Malay,English": 0.6666666666666666, "Indonesian,English": 0.6466666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.31333333333333335, "Filipino,Vietnamese,Spanish": 0.32666666666666666, "Filipino,Vietnamese,Malay": 0.32, "Filipino,Vietnamese,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,English": 0.31333333333333335, "Filipino,Chinese,Spanish": 0.34, "Filipino,Chinese,Malay": 0.32, "Filipino,Chinese,Indonesian": 0.31333333333333335, "Filipino,Chinese,English": 0.32, "Filipino,Spanish,Malay": 0.3333333333333333, "Filipino,Spanish,Indonesian": 0.32, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.31333333333333335, "Filipino,Malay,English": 0.31333333333333335, "Filipino,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish": 0.5266666666666666, "Vietnamese,Chinese,Malay": 0.52, "Vietnamese,Chinese,Indonesian": 0.5266666666666666, "Vietnamese,Chinese,English": 0.5333333333333333, "Vietnamese,Spanish,Malay": 0.5333333333333333, "Vietnamese,Spanish,Indonesian": 0.5266666666666666, "Vietnamese,Spanish,English": 0.5266666666666666, "Vietnamese,Malay,Indonesian": 0.5533333333333333, "Vietnamese,Malay,English": 0.5266666666666666, "Vietnamese,Indonesian,English": 0.5266666666666666, "Chinese,Spanish,Malay": 0.5133333333333333, "Chinese,Spanish,Indonesian": 0.52, "Chinese,Spanish,English": 0.5333333333333333, "Chinese,Malay,Indonesian": 0.5133333333333333, "Chinese,Malay,English": 0.5133333333333333, "Chinese,Indonesian,English": 0.5266666666666666, "Spanish,Malay,Indonesian": 0.5533333333333333, "Spanish,Malay,English": 0.5333333333333333, "Spanish,Indonesian,English": 0.52, "Malay,Indonesian,English": 0.5466666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,English": 0.26, "Filipino,Vietnamese,Spanish,Malay": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay": 0.28, "Filipino,Chinese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,English": 0.28, "Filipino,Chinese,Malay,Indonesian": 0.2733333333333333, "Filipino,Chinese,Malay,English": 0.2733333333333333, "Filipino,Chinese,Indonesian,English": 0.2733333333333333, "Filipino,Spanish,Malay,Indonesian": 0.2733333333333333, "Filipino,Spanish,Malay,English": 0.26666666666666666, "Filipino,Spanish,Indonesian,English": 0.26, "Filipino,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.44, "Vietnamese,Chinese,Spanish,Indonesian": 0.44, "Vietnamese,Chinese,Spanish,English": 0.44666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.44, "Vietnamese,Chinese,Malay,English": 0.44, "Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Spanish,Malay,English": 0.4533333333333333, "Vietnamese,Spanish,Indonesian,English": 0.46, "Vietnamese,Malay,Indonesian,English": 0.46, "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Chinese,Spanish,Malay,English": 0.44, "Chinese,Spanish,Indonesian,English": 0.44666666666666666, "Chinese,Malay,Indonesian,English": 0.44666666666666666, "Spanish,Malay,Indonesian,English": 0.47333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.24, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.24, "Filipino,Chinese,Spanish,Malay,English": 0.24, "Filipino,Chinese,Spanish,Indonesian,English": 0.24, "Filipino,Chinese,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4, "Vietnamese,Chinese,Malay,Indonesian,English": 0.3933333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.42, "Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 } }, "AC3_2": 0.565595437901492, "AC3_3": 0.4825504753966382, "AC3_4": 0.4199598628687718, "AC3_5": 0.3687090686639471, "AC3_6": 0.32423909832330516, "AC3_7": 0.28419495209333273 }, "prompt_4": { "overall_acc": 0.5333333333333333, "language_acc": { "Filipino": 0.3933333333333333, "Vietnamese": 0.5133333333333333, "Chinese": 0.58, "Spanish": 0.5333333333333333, "Malay": 0.5533333333333333, "Indonesian": 0.5133333333333333, "English": 0.6466666666666666 }, "consistency_score_2": 0.6044444444444443, "consistency_score_3": 0.44876190476190475, "consistency_score_4": 0.36095238095238097, "consistency_score_5": 0.3028571428571429, "consistency_score_6": 0.26, "consistency_score_7": 0.22666666666666666, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4266666666666667, "Filipino,Chinese": 0.46, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.44, "Filipino,Indonesian": 0.44, "Filipino,English": 0.46, "Vietnamese,Chinese": 0.7066666666666667, "Vietnamese,Spanish": 0.64, "Vietnamese,Malay": 0.6533333333333333, "Vietnamese,Indonesian": 0.6533333333333333, "Vietnamese,English": 0.6666666666666666, "Chinese,Spanish": 0.64, "Chinese,Malay": 0.66, "Chinese,Indonesian": 0.6533333333333333, "Chinese,English": 0.6733333333333333, "Spanish,Malay": 0.66, "Spanish,Indonesian": 0.6666666666666666, "Spanish,English": 0.68, "Malay,Indonesian": 0.7066666666666667, "Malay,English": 0.6733333333333333, "Indonesian,English": 0.6533333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.3333333333333333, "Filipino,Vietnamese,Spanish": 0.34, "Filipino,Vietnamese,Malay": 0.32, "Filipino,Vietnamese,Indonesian": 0.32, "Filipino,Vietnamese,English": 0.34, "Filipino,Chinese,Spanish": 0.34, "Filipino,Chinese,Malay": 0.3333333333333333, "Filipino,Chinese,Indonesian": 0.31333333333333335, "Filipino,Chinese,English": 0.35333333333333333, "Filipino,Spanish,Malay": 0.34, "Filipino,Spanish,Indonesian": 0.3466666666666667, "Filipino,Spanish,English": 0.37333333333333335, "Filipino,Malay,Indonesian": 0.35333333333333333, "Filipino,Malay,English": 0.3333333333333333, "Filipino,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish": 0.5333333333333333, "Vietnamese,Chinese,Malay": 0.54, "Vietnamese,Chinese,Indonesian": 0.54, "Vietnamese,Chinese,English": 0.54, "Vietnamese,Spanish,Malay": 0.5133333333333333, "Vietnamese,Spanish,Indonesian": 0.52, "Vietnamese,Spanish,English": 0.52, "Vietnamese,Malay,Indonesian": 0.5333333333333333, "Vietnamese,Malay,English": 0.5133333333333333, "Vietnamese,Indonesian,English": 0.52, "Chinese,Spanish,Malay": 0.5266666666666666, "Chinese,Spanish,Indonesian": 0.5266666666666666, "Chinese,Spanish,English": 0.5266666666666666, "Chinese,Malay,Indonesian": 0.5533333333333333, "Chinese,Malay,English": 0.5333333333333333, "Chinese,Indonesian,English": 0.5266666666666666, "Spanish,Malay,Indonesian": 0.5466666666666666, "Spanish,Malay,English": 0.5333333333333333, "Spanish,Indonesian,English": 0.54, "Malay,Indonesian,English": 0.5466666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.28, "Filipino,Vietnamese,Chinese,Malay": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,English": 0.2866666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.28, "Filipino,Vietnamese,Spanish,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,English": 0.30666666666666664, "Filipino,Vietnamese,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Malay,English": 0.2733333333333333, "Filipino,Vietnamese,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay": 0.29333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.2866666666666667, "Filipino,Chinese,Spanish,English": 0.3, "Filipino,Chinese,Malay,Indonesian": 0.29333333333333333, "Filipino,Chinese,Malay,English": 0.2866666666666667, "Filipino,Chinese,Indonesian,English": 0.29333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.29333333333333333, "Filipino,Spanish,Malay,English": 0.30666666666666664, "Filipino,Spanish,Indonesian,English": 0.30666666666666664, "Filipino,Malay,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.4533333333333333, "Vietnamese,Chinese,Spanish,English": 0.44666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,Malay,English": 0.44, "Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.4533333333333333, "Vietnamese,Spanish,Malay,English": 0.43333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.44666666666666666, "Vietnamese,Malay,Indonesian,English": 0.44666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.4666666666666667, "Chinese,Spanish,Malay,English": 0.4533333333333333, "Chinese,Spanish,Indonesian,English": 0.4666666666666667, "Chinese,Malay,Indonesian,English": 0.48, "Spanish,Malay,Indonesian,English": 0.48 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.26, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,English": 0.26, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.26, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.26666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.28, "Filipino,Spanish,Malay,Indonesian,English": 0.2866666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4066666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.4066666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4, "Chinese,Spanish,Malay,Indonesian,English": 0.4266666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36666666666666664 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 } }, "AC3_2": 0.5666666666168619, "AC3_3": 0.4874062580315706, "AC3_4": 0.4305289314392557, "AC3_5": 0.3863325739856892, "AC3_6": 0.3495798318887084, "AC3_7": 0.3181286549289012 }, "prompt_5": { "overall_acc": 0.5361904761904762, "language_acc": { "Filipino": 0.3933333333333333, "Vietnamese": 0.52, "Chinese": 0.6, "Spanish": 0.5266666666666666, "Malay": 0.5333333333333333, "Indonesian": 0.54, "English": 0.64 }, "consistency_score_2": 0.6215873015873016, "consistency_score_3": 0.4666666666666666, "consistency_score_4": 0.37428571428571433, "consistency_score_5": 0.31015873015873013, "consistency_score_6": 0.26095238095238094, "consistency_score_7": 0.22, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4, "Filipino,Chinese": 0.49333333333333335, "Filipino,Spanish": 0.4666666666666667, "Filipino,Malay": 0.4533333333333333, "Filipino,Indonesian": 0.4533333333333333, "Filipino,English": 0.5133333333333333, "Vietnamese,Chinese": 0.6666666666666666, "Vietnamese,Spanish": 0.66, "Vietnamese,Malay": 0.64, "Vietnamese,Indonesian": 0.6666666666666666, "Vietnamese,English": 0.68, "Chinese,Spanish": 0.6466666666666666, "Chinese,Malay": 0.6733333333333333, "Chinese,Indonesian": 0.66, "Chinese,English": 0.6666666666666666, "Spanish,Malay": 0.6733333333333333, "Spanish,Indonesian": 0.7133333333333334, "Spanish,English": 0.6866666666666666, "Malay,Indonesian": 0.7533333333333333, "Malay,English": 0.7266666666666667, "Indonesian,English": 0.76 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.31333333333333335, "Filipino,Vietnamese,Spanish": 0.31333333333333335, "Filipino,Vietnamese,Malay": 0.31333333333333335, "Filipino,Vietnamese,Indonesian": 0.32, "Filipino,Vietnamese,English": 0.34, "Filipino,Chinese,Spanish": 0.35333333333333333, "Filipino,Chinese,Malay": 0.36666666666666664, "Filipino,Chinese,Indonesian": 0.3466666666666667, "Filipino,Chinese,English": 0.38, "Filipino,Spanish,Malay": 0.3466666666666667, "Filipino,Spanish,Indonesian": 0.35333333333333333, "Filipino,Spanish,English": 0.36666666666666664, "Filipino,Malay,Indonesian": 0.36, "Filipino,Malay,English": 0.36666666666666664, "Filipino,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Spanish": 0.52, "Vietnamese,Chinese,Malay": 0.52, "Vietnamese,Chinese,Indonesian": 0.5333333333333333, "Vietnamese,Chinese,English": 0.5266666666666666, "Vietnamese,Spanish,Malay": 0.52, "Vietnamese,Spanish,Indonesian": 0.5533333333333333, "Vietnamese,Spanish,English": 0.5333333333333333, "Vietnamese,Malay,Indonesian": 0.5533333333333333, "Vietnamese,Malay,English": 0.5533333333333333, "Vietnamese,Indonesian,English": 0.5866666666666667, "Chinese,Spanish,Malay": 0.5266666666666666, "Chinese,Spanish,Indonesian": 0.5466666666666666, "Chinese,Spanish,English": 0.5266666666666666, "Chinese,Malay,Indonesian": 0.5666666666666667, "Chinese,Malay,English": 0.56, "Chinese,Indonesian,English": 0.5733333333333334, "Spanish,Malay,Indonesian": 0.6, "Spanish,Malay,English": 0.5666666666666667, "Spanish,Indonesian,English": 0.5933333333333334, "Malay,Indonesian,English": 0.64 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.26, "Filipino,Vietnamese,Chinese,Malay": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,English": 0.28, "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Spanish,English": 0.2866666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Malay,English": 0.2866666666666667, "Filipino,Vietnamese,Indonesian,English": 0.31333333333333335, "Filipino,Chinese,Spanish,Malay": 0.29333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.3, "Filipino,Chinese,Spanish,English": 0.32, "Filipino,Chinese,Malay,Indonesian": 0.30666666666666664, "Filipino,Chinese,Malay,English": 0.32666666666666666, "Filipino,Chinese,Indonesian,English": 0.3333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.3, "Filipino,Spanish,Malay,English": 0.31333333333333335, "Filipino,Spanish,Indonesian,English": 0.32666666666666666, "Filipino,Malay,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.44, "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, "Vietnamese,Chinese,Spanish,English": 0.44, "Vietnamese,Chinese,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,Malay,English": 0.44666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.48, "Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Spanish,Malay,English": 0.46, "Vietnamese,Spanish,Indonesian,English": 0.49333333333333335, "Vietnamese,Malay,Indonesian,English": 0.5066666666666667, "Chinese,Spanish,Malay,Indonesian": 0.4866666666666667, "Chinese,Spanish,Malay,English": 0.4666666666666667, "Chinese,Spanish,Indonesian,English": 0.4866666666666667, "Chinese,Malay,Indonesian,English": 0.5066666666666667, "Spanish,Malay,Indonesian,English": 0.5333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Spanish,English": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.28, "Filipino,Chinese,Spanish,Indonesian,English": 0.29333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.3, "Filipino,Spanish,Malay,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.3933333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4266666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.4266666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.44666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 } }, "AC3_2": 0.5757394857637913, "AC3_3": 0.49901867674669487, "AC3_4": 0.4408427973215751, "AC3_5": 0.3929918193369835, "AC3_6": 0.3510542185370612, "AC3_7": 0.31198992439199125 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.44724025974025977, "language_acc": { "Indonesian": 0.42613636363636365, "English": 0.5056818181818182, "Filipino": 0.3409090909090909, "Spanish": 0.3977272727272727, "Chinese": 0.5738636363636364, "Malay": 0.4772727272727273, "Vietnamese": 0.4090909090909091 }, "consistency_score_2": 0.518939393939394, "consistency_score_3": 0.34480519480519484, "consistency_score_4": 0.2512987012987013, "consistency_score_5": 0.191017316017316, "consistency_score_6": 0.1477272727272727, "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5852272727272727, "Indonesian,Filipino": 0.38636363636363635, "Indonesian,Spanish": 0.6363636363636364, "Indonesian,Chinese": 0.5681818181818182, "Indonesian,Malay": 0.6761363636363636, "Indonesian,Vietnamese": 0.5965909090909091, "English,Filipino": 0.375, "English,Spanish": 0.5113636363636364, "English,Chinese": 0.6079545454545454, "English,Malay": 0.5568181818181818, "English,Vietnamese": 0.5284090909090909, "Filipino,Spanish": 0.4318181818181818, "Filipino,Chinese": 0.3522727272727273, "Filipino,Malay": 0.42045454545454547, "Filipino,Vietnamese": 0.3465909090909091, "Spanish,Chinese": 0.4659090909090909, "Spanish,Malay": 0.6193181818181818, "Spanish,Vietnamese": 0.5227272727272727, "Chinese,Malay": 0.5852272727272727, "Chinese,Vietnamese": 0.5454545454545454, "Malay,Vietnamese": 0.5795454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.26704545454545453, "Indonesian,English,Spanish": 0.42045454545454547, "Indonesian,English,Chinese": 0.4431818181818182, "Indonesian,English,Malay": 0.44886363636363635, "Indonesian,English,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Spanish": 0.30113636363636365, "Indonesian,Filipino,Chinese": 0.23295454545454544, "Indonesian,Filipino,Malay": 0.29545454545454547, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.38636363636363635, "Indonesian,Spanish,Malay": 0.5113636363636364, "Indonesian,Spanish,Vietnamese": 0.4147727272727273, "Indonesian,Chinese,Malay": 0.44886363636363635, "Indonesian,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Spanish": 0.25, "English,Filipino,Chinese": 0.26136363636363635, "English,Filipino,Malay": 0.2784090909090909, "English,Filipino,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese": 0.35795454545454547, "English,Spanish,Malay": 0.38636363636363635, "English,Spanish,Vietnamese": 0.3522727272727273, "English,Chinese,Malay": 0.4318181818181818, "English,Chinese,Vietnamese": 0.3977272727272727, "English,Malay,Vietnamese": 0.38636363636363635, "Filipino,Spanish,Chinese": 0.2159090909090909, "Filipino,Spanish,Malay": 0.3068181818181818, "Filipino,Spanish,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay": 0.25, "Filipino,Chinese,Vietnamese": 0.2215909090909091, "Filipino,Malay,Vietnamese": 0.25, "Spanish,Chinese,Malay": 0.3806818181818182, "Spanish,Chinese,Vietnamese": 0.32386363636363635, "Spanish,Malay,Vietnamese": 0.4147727272727273, "Chinese,Malay,Vietnamese": 0.4090909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2159090909090909, "Indonesian,English,Filipino,Chinese": 0.19886363636363635, "Indonesian,English,Filipino,Malay": 0.23295454545454544, "Indonesian,English,Filipino,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Chinese": 0.32954545454545453, "Indonesian,English,Spanish,Malay": 0.36363636363636365, "Indonesian,English,Spanish,Vietnamese": 0.3125, "Indonesian,English,Chinese,Malay": 0.36363636363636365, "Indonesian,English,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,English,Malay,Vietnamese": 0.3409090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Malay": 0.25, "Indonesian,Filipino,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Chinese,Malay": 0.19318181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Chinese,Malay": 0.3352272727272727, "Indonesian,Spanish,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.3409090909090909, "English,Filipino,Spanish,Chinese": 0.17045454545454544, "English,Filipino,Spanish,Malay": 0.21022727272727273, "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay": 0.21022727272727273, "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Malay,Vietnamese": 0.1875, "English,Spanish,Chinese,Malay": 0.3125, "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "English,Spanish,Malay,Vietnamese": 0.29545454545454547, "English,Chinese,Malay,Vietnamese": 0.3181818181818182, "Filipino,Spanish,Chinese,Malay": 0.1875, "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.2897727272727273 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.29545454545454547, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1875, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, "English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, "AC3_2": 0.4804294696684598, "AC3_3": 0.38939877498191305, "AC3_4": 0.3217884833738374, "AC3_5": 0.26769955347787405, "AC3_6": 0.22209475377638804, "AC3_7": 0.18122615442105017 }, "prompt_2": { "overall_acc": 0.461038961038961, "language_acc": { "Indonesian": 0.44886363636363635, "English": 0.5113636363636364, "Filipino": 0.3409090909090909, "Spanish": 0.4034090909090909, "Chinese": 0.5852272727272727, "Malay": 0.4943181818181818, "Vietnamese": 0.4431818181818182 }, "consistency_score_2": 0.5192099567099567, "consistency_score_3": 0.3487012987012987, "consistency_score_4": 0.2581168831168831, "consistency_score_5": 0.2015692640692641, "consistency_score_6": 0.16152597402597405, "consistency_score_7": 0.13068181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6420454545454546, "Indonesian,Filipino": 0.3465909090909091, "Indonesian,Spanish": 0.5909090909090909, "Indonesian,Chinese": 0.5454545454545454, "Indonesian,Malay": 0.6420454545454546, "Indonesian,Vietnamese": 0.5738636363636364, "English,Filipino": 0.3977272727272727, "English,Spanish": 0.5852272727272727, "English,Chinese": 0.5909090909090909, "English,Malay": 0.6363636363636364, "English,Vietnamese": 0.5625, "Filipino,Spanish": 0.4090909090909091, "Filipino,Chinese": 0.3465909090909091, "Filipino,Malay": 0.375, "Filipino,Vietnamese": 0.36363636363636365, "Spanish,Chinese": 0.5284090909090909, "Spanish,Malay": 0.6079545454545454, "Spanish,Vietnamese": 0.5340909090909091, "Chinese,Malay": 0.5909090909090909, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.5170454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.26704545454545453, "Indonesian,English,Spanish": 0.45454545454545453, "Indonesian,English,Chinese": 0.44886363636363635, "Indonesian,English,Malay": 0.5113636363636364, "Indonesian,English,Vietnamese": 0.4431818181818182, "Indonesian,Filipino,Spanish": 0.2840909090909091, "Indonesian,Filipino,Chinese": 0.22727272727272727, "Indonesian,Filipino,Malay": 0.26136363636363635, "Indonesian,Filipino,Vietnamese": 0.23295454545454544, "Indonesian,Spanish,Chinese": 0.38636363636363635, "Indonesian,Spanish,Malay": 0.48295454545454547, "Indonesian,Spanish,Vietnamese": 0.4034090909090909, "Indonesian,Chinese,Malay": 0.42613636363636365, "Indonesian,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,Malay,Vietnamese": 0.42613636363636365, "English,Filipino,Spanish": 0.29545454545454547, "English,Filipino,Chinese": 0.23863636363636365, "English,Filipino,Malay": 0.2784090909090909, "English,Filipino,Vietnamese": 0.23295454545454544, "English,Spanish,Chinese": 0.3977272727272727, "English,Spanish,Malay": 0.4375, "English,Spanish,Vietnamese": 0.3977272727272727, "English,Chinese,Malay": 0.45454545454545453, "English,Chinese,Vietnamese": 0.4034090909090909, "English,Malay,Vietnamese": 0.4147727272727273, "Filipino,Spanish,Chinese": 0.24431818181818182, "Filipino,Spanish,Malay": 0.2784090909090909, "Filipino,Spanish,Vietnamese": 0.25, "Filipino,Chinese,Malay": 0.24431818181818182, "Filipino,Chinese,Vietnamese": 0.2159090909090909, "Filipino,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay": 0.42045454545454547, "Spanish,Chinese,Vietnamese": 0.3522727272727273, "Spanish,Malay,Vietnamese": 0.39204545454545453, "Chinese,Malay,Vietnamese": 0.375 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.23863636363636365, "Indonesian,English,Filipino,Chinese": 0.1875, "Indonesian,English,Filipino,Malay": 0.22727272727272727, "Indonesian,English,Filipino,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Chinese": 0.3465909090909091, "Indonesian,English,Spanish,Malay": 0.3977272727272727, "Indonesian,English,Spanish,Vietnamese": 0.3352272727272727, "Indonesian,English,Chinese,Malay": 0.38636363636363635, "Indonesian,English,Chinese,Vietnamese": 0.3522727272727273, "Indonesian,English,Malay,Vietnamese": 0.36363636363636365, "Indonesian,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Chinese,Malay": 0.19886363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Spanish,Chinese,Malay": 0.3409090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.30113636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.3465909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.32954545454545453, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.2215909090909091, "English,Filipino,Spanish,Vietnamese": 0.19886363636363635, "English,Filipino,Chinese,Malay": 0.19886363636363635, "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "English,Filipino,Malay,Vietnamese": 0.17613636363636365, "English,Spanish,Chinese,Malay": 0.3352272727272727, "English,Spanish,Chinese,Vietnamese": 0.29545454545454547, "English,Spanish,Malay,Vietnamese": 0.3181818181818182, "English,Chinese,Malay,Vietnamese": 0.32954545454545453, "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.17613636363636365, "Indonesian,English,Filipino,Spanish,Malay": 0.21022727272727273, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Spanish,Chinese,Malay": 0.3125, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.30113636363636365, "Indonesian,English,Chinese,Malay,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1875, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.25, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 } }, "AC3_2": 0.4883984356298704, "AC3_3": 0.39707766166374836, "AC3_4": 0.33094896069978136, "AC3_5": 0.2805014503243686, "AC3_6": 0.23923534089182885, "AC3_7": 0.2036413517550848 }, "prompt_3": { "overall_acc": 0.45698051948051954, "language_acc": { "Indonesian": 0.4375, "English": 0.48863636363636365, "Filipino": 0.3409090909090909, "Spanish": 0.42613636363636365, "Chinese": 0.5795454545454546, "Malay": 0.5056818181818182, "Vietnamese": 0.42045454545454547 }, "consistency_score_2": 0.5211038961038961, "consistency_score_3": 0.34512987012987006, "consistency_score_4": 0.2506493506493506, "consistency_score_5": 0.18966450216450217, "consistency_score_6": 0.1461038961038961, "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5795454545454546, "Indonesian,Filipino": 0.36363636363636365, "Indonesian,Spanish": 0.6079545454545454, "Indonesian,Chinese": 0.5625, "Indonesian,Malay": 0.6590909090909091, "Indonesian,Vietnamese": 0.6022727272727273, "English,Filipino": 0.3977272727272727, "English,Spanish": 0.5965909090909091, "English,Chinese": 0.5852272727272727, "English,Malay": 0.6022727272727273, "English,Vietnamese": 0.5397727272727273, "Filipino,Spanish": 0.4375, "Filipino,Chinese": 0.36363636363636365, "Filipino,Malay": 0.38636363636363635, "Filipino,Vietnamese": 0.3806818181818182, "Spanish,Chinese": 0.5511363636363636, "Spanish,Malay": 0.6079545454545454, "Spanish,Vietnamese": 0.4772727272727273, "Chinese,Malay": 0.5738636363636364, "Chinese,Vietnamese": 0.5284090909090909, "Malay,Vietnamese": 0.5397727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.24431818181818182, "Indonesian,English,Spanish": 0.4431818181818182, "Indonesian,English,Chinese": 0.4090909090909091, "Indonesian,English,Malay": 0.4659090909090909, "Indonesian,English,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish": 0.2840909090909091, "Indonesian,Filipino,Chinese": 0.22727272727272727, "Indonesian,Filipino,Malay": 0.2727272727272727, "Indonesian,Filipino,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Chinese": 0.42045454545454547, "Indonesian,Spanish,Malay": 0.48863636363636365, "Indonesian,Spanish,Vietnamese": 0.4090909090909091, "Indonesian,Chinese,Malay": 0.42613636363636365, "Indonesian,Chinese,Vietnamese": 0.42045454545454547, "Indonesian,Malay,Vietnamese": 0.45454545454545453, "English,Filipino,Spanish": 0.2727272727272727, "English,Filipino,Chinese": 0.23295454545454544, "English,Filipino,Malay": 0.2784090909090909, "English,Filipino,Vietnamese": 0.23295454545454544, "English,Spanish,Chinese": 0.42045454545454547, "English,Spanish,Malay": 0.4318181818181818, "English,Spanish,Vietnamese": 0.36363636363636365, "English,Chinese,Malay": 0.42613636363636365, "English,Chinese,Vietnamese": 0.3806818181818182, "English,Malay,Vietnamese": 0.39204545454545453, "Filipino,Spanish,Chinese": 0.26136363636363635, "Filipino,Spanish,Malay": 0.2897727272727273, "Filipino,Spanish,Vietnamese": 0.23295454545454544, "Filipino,Chinese,Malay": 0.23863636363636365, "Filipino,Chinese,Vietnamese": 0.23863636363636365, "Filipino,Malay,Vietnamese": 0.23863636363636365, "Spanish,Chinese,Malay": 0.42613636363636365, "Spanish,Chinese,Vietnamese": 0.3465909090909091, "Spanish,Malay,Vietnamese": 0.3693181818181818, "Chinese,Malay,Vietnamese": 0.3693181818181818 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.20454545454545456, "Indonesian,English,Filipino,Chinese": 0.1590909090909091, "Indonesian,English,Filipino,Malay": 0.21022727272727273, "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese": 0.3409090909090909, "Indonesian,English,Spanish,Malay": 0.36363636363636365, "Indonesian,English,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,English,Chinese,Malay": 0.3465909090909091, "Indonesian,English,Chinese,Vietnamese": 0.3125, "Indonesian,English,Malay,Vietnamese": 0.3409090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Malay": 0.22727272727272727, "Indonesian,Filipino,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Chinese,Malay": 0.1875, "Indonesian,Filipino,Chinese,Vietnamese": 0.1875, "Indonesian,Filipino,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Chinese,Malay": 0.3522727272727273, "Indonesian,Spanish,Chinese,Vietnamese": 0.3181818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.3409090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.3352272727272727, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.2159090909090909, "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "English,Filipino,Chinese,Malay": 0.18181818181818182, "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Malay,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese,Malay": 0.3465909090909091, "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, "English,Spanish,Malay,Vietnamese": 0.29545454545454547, "English,Chinese,Malay,Vietnamese": 0.3068181818181818, "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.29545454545454547, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.26136363636363635, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2727272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.125, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, "AC3_2": 0.4869402381352664, "AC3_3": 0.3932566623333177, "AC3_4": 0.3237338481203083, "AC3_5": 0.2680697440220125, "AC3_6": 0.2214172099403882, "AC3_7": 0.18201215566446877 }, "prompt_4": { "overall_acc": 0.4488636363636364, "language_acc": { "Indonesian": 0.44886363636363635, "English": 0.5, "Filipino": 0.32386363636363635, "Spanish": 0.42045454545454547, "Chinese": 0.5625, "Malay": 0.48863636363636365, "Vietnamese": 0.3977272727272727 }, "consistency_score_2": 0.5186688311688312, "consistency_score_3": 0.3362012987012988, "consistency_score_4": 0.23522727272727276, "consistency_score_5": 0.16937229437229437, "consistency_score_6": 0.12337662337662338, "consistency_score_7": 0.09090909090909091, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6136363636363636, "Indonesian,Filipino": 0.375, "Indonesian,Spanish": 0.6363636363636364, "Indonesian,Chinese": 0.5340909090909091, "Indonesian,Malay": 0.6931818181818182, "Indonesian,Vietnamese": 0.5568181818181818, "English,Filipino": 0.38636363636363635, "English,Spanish": 0.5681818181818182, "English,Chinese": 0.5568181818181818, "English,Malay": 0.6420454545454546, "English,Vietnamese": 0.5113636363636364, "Filipino,Spanish": 0.3465909090909091, "Filipino,Chinese": 0.3806818181818182, "Filipino,Malay": 0.3977272727272727, "Filipino,Vietnamese": 0.3352272727272727, "Spanish,Chinese": 0.48295454545454547, "Spanish,Malay": 0.6534090909090909, "Spanish,Vietnamese": 0.5681818181818182, "Chinese,Malay": 0.5795454545454546, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.5568181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.2727272727272727, "Indonesian,English,Spanish": 0.4375, "Indonesian,English,Chinese": 0.3977272727272727, "Indonesian,English,Malay": 0.4943181818181818, "Indonesian,English,Vietnamese": 0.375, "Indonesian,Filipino,Spanish": 0.26704545454545453, "Indonesian,Filipino,Chinese": 0.24431818181818182, "Indonesian,Filipino,Malay": 0.2897727272727273, "Indonesian,Filipino,Vietnamese": 0.23295454545454544, "Indonesian,Spanish,Chinese": 0.36363636363636365, "Indonesian,Spanish,Malay": 0.5227272727272727, "Indonesian,Spanish,Vietnamese": 0.42613636363636365, "Indonesian,Chinese,Malay": 0.42613636363636365, "Indonesian,Chinese,Vietnamese": 0.35795454545454547, "Indonesian,Malay,Vietnamese": 0.4318181818181818, "English,Filipino,Spanish": 0.23295454545454544, "English,Filipino,Chinese": 0.23295454545454544, "English,Filipino,Malay": 0.2784090909090909, "English,Filipino,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese": 0.36363636363636365, "English,Spanish,Malay": 0.45454545454545453, "English,Spanish,Vietnamese": 0.3806818181818182, "English,Chinese,Malay": 0.42613636363636365, "English,Chinese,Vietnamese": 0.3522727272727273, "English,Malay,Vietnamese": 0.38636363636363635, "Filipino,Spanish,Chinese": 0.21022727272727273, "Filipino,Spanish,Malay": 0.2784090909090909, "Filipino,Spanish,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay": 0.25, "Filipino,Chinese,Vietnamese": 0.2159090909090909, "Filipino,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay": 0.3977272727272727, "Spanish,Chinese,Vietnamese": 0.3352272727272727, "Spanish,Malay,Vietnamese": 0.4375, "Chinese,Malay,Vietnamese": 0.36363636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.19318181818181818, "Indonesian,English,Filipino,Chinese": 0.19318181818181818, "Indonesian,English,Filipino,Malay": 0.2215909090909091, "Indonesian,English,Filipino,Vietnamese": 0.17045454545454544, "Indonesian,English,Spanish,Chinese": 0.29545454545454547, "Indonesian,English,Spanish,Malay": 0.3806818181818182, "Indonesian,English,Spanish,Vietnamese": 0.30113636363636365, "Indonesian,English,Chinese,Malay": 0.3352272727272727, "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,English,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Malay": 0.23295454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Chinese,Malay": 0.21022727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese,Malay": 0.3181818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.3693181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.2840909090909091, "English,Filipino,Spanish,Chinese": 0.1534090909090909, "English,Filipino,Spanish,Malay": 0.19886363636363635, "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, "English,Filipino,Chinese,Malay": 0.19318181818181818, "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, "English,Filipino,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay": 0.3181818181818182, "English,Spanish,Chinese,Vietnamese": 0.26136363636363635, "English,Spanish,Malay,Vietnamese": 0.3125, "English,Chinese,Malay,Vietnamese": 0.2840909090909091, "Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.1875, "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.125, "Indonesian,English,Filipino,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Spanish,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.26704545454545453, "Indonesian,English,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 } }, "AC3_2": 0.48124809329986373, "AC3_3": 0.3844485487140946, "AC3_4": 0.30868695253717776, "AC3_5": 0.24594191362642126, "AC3_6": 0.19355254671021244, "AC3_7": 0.15119617222079276 }, "prompt_5": { "overall_acc": 0.44724025974025977, "language_acc": { "Indonesian": 0.42613636363636365, "English": 0.4715909090909091, "Filipino": 0.3409090909090909, "Spanish": 0.4318181818181818, "Chinese": 0.5738636363636364, "Malay": 0.4772727272727273, "Vietnamese": 0.4090909090909091 }, "consistency_score_2": 0.533279220779221, "consistency_score_3": 0.3573051948051947, "consistency_score_4": 0.26022727272727275, "consistency_score_5": 0.1966991341991342, "consistency_score_6": 0.15178571428571427, "consistency_score_7": 0.11931818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6079545454545454, "Indonesian,Filipino": 0.3465909090909091, "Indonesian,Spanish": 0.6477272727272727, "Indonesian,Chinese": 0.5397727272727273, "Indonesian,Malay": 0.6818181818181818, "Indonesian,Vietnamese": 0.6306818181818182, "English,Filipino": 0.38636363636363635, "English,Spanish": 0.6306818181818182, "English,Chinese": 0.5625, "English,Malay": 0.6306818181818182, "English,Vietnamese": 0.5284090909090909, "Filipino,Spanish": 0.38636363636363635, "Filipino,Chinese": 0.35795454545454547, "Filipino,Malay": 0.4034090909090909, "Filipino,Vietnamese": 0.3409090909090909, "Spanish,Chinese": 0.5568181818181818, "Spanish,Malay": 0.6420454545454546, "Spanish,Vietnamese": 0.5795454545454546, "Chinese,Malay": 0.5909090909090909, "Chinese,Vietnamese": 0.5681818181818182, "Malay,Vietnamese": 0.5795454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.24431818181818182, "Indonesian,English,Spanish": 0.4659090909090909, "Indonesian,English,Chinese": 0.4147727272727273, "Indonesian,English,Malay": 0.48863636363636365, "Indonesian,English,Vietnamese": 0.4318181818181818, "Indonesian,Filipino,Spanish": 0.2556818181818182, "Indonesian,Filipino,Chinese": 0.21022727272727273, "Indonesian,Filipino,Malay": 0.2784090909090909, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.4090909090909091, "Indonesian,Spanish,Malay": 0.5056818181818182, "Indonesian,Spanish,Vietnamese": 0.4715909090909091, "Indonesian,Chinese,Malay": 0.4431818181818182, "Indonesian,Chinese,Vietnamese": 0.42045454545454547, "Indonesian,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Spanish": 0.2556818181818182, "English,Filipino,Chinese": 0.24431818181818182, "English,Filipino,Malay": 0.2727272727272727, "English,Filipino,Vietnamese": 0.2215909090909091, "English,Spanish,Chinese": 0.4375, "English,Spanish,Malay": 0.48295454545454547, "English,Spanish,Vietnamese": 0.4147727272727273, "English,Chinese,Malay": 0.44886363636363635, "English,Chinese,Vietnamese": 0.38636363636363635, "English,Malay,Vietnamese": 0.4147727272727273, "Filipino,Spanish,Chinese": 0.23863636363636365, "Filipino,Spanish,Malay": 0.2840909090909091, "Filipino,Spanish,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay": 0.24431818181818182, "Filipino,Chinese,Vietnamese": 0.2215909090909091, "Filipino,Malay,Vietnamese": 0.23863636363636365, "Spanish,Chinese,Malay": 0.4375, "Spanish,Chinese,Vietnamese": 0.4090909090909091, "Spanish,Malay,Vietnamese": 0.4375, "Chinese,Malay,Vietnamese": 0.4147727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1875, "Indonesian,English,Filipino,Chinese": 0.18181818181818182, "Indonesian,English,Filipino,Malay": 0.20454545454545456, "Indonesian,English,Filipino,Vietnamese": 0.1875, "Indonesian,English,Spanish,Chinese": 0.3465909090909091, "Indonesian,English,Spanish,Malay": 0.39204545454545453, "Indonesian,English,Spanish,Vietnamese": 0.35795454545454547, "Indonesian,English,Chinese,Malay": 0.36363636363636365, "Indonesian,English,Chinese,Vietnamese": 0.32954545454545453, "Indonesian,English,Malay,Vietnamese": 0.375, "Indonesian,Filipino,Spanish,Chinese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.1875, "Indonesian,Filipino,Chinese,Malay": 0.1875, "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese,Malay": 0.3522727272727273, "Indonesian,Spanish,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.39204545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish,Chinese": 0.19886363636363635, "English,Filipino,Spanish,Malay": 0.21022727272727273, "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay": 0.20454545454545456, "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Malay,Vietnamese": 0.1875, "English,Spanish,Chinese,Malay": 0.3806818181818182, "English,Spanish,Chinese,Vietnamese": 0.3181818181818182, "English,Spanish,Malay,Vietnamese": 0.3352272727272727, "English,Chinese,Malay,Vietnamese": 0.32954545454545453, "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.1875, "Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.3125, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3125, "Indonesian,English,Chinese,Malay,Vietnamese": 0.29545454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 } }, "AC3_2": 0.4864848520188464, "AC3_3": 0.39724609022032276, "AC3_4": 0.32901612494827875, "AC3_5": 0.2732299737652739, "AC3_6": 0.2266502128928643, "AC3_7": 0.1883791351582064 } }, "sg_eval": { "prompt_1": { "accuracy": 0.5631067961165048 }, "prompt_2": { "accuracy": 0.5922330097087378 }, "prompt_3": { "accuracy": 0.5922330097087378 }, "prompt_4": { "accuracy": 0.5436893203883495 }, "prompt_5": { "accuracy": 0.5922330097087378 } }, "cn_eval": { "prompt_1": { "accuracy": 0.5428571428571428 }, "prompt_2": { "accuracy": 0.6095238095238096 }, "prompt_3": { "accuracy": 0.580952380952381 }, "prompt_4": { "accuracy": 0.5428571428571428 }, "prompt_5": { "accuracy": 0.6095238095238096 } }, "us_eval": { "prompt_1": { "accuracy": 0.5794392523364486 }, "prompt_2": { "accuracy": 0.5420560747663551 }, "prompt_3": { "accuracy": 0.514018691588785 }, "prompt_4": { "accuracy": 0.5700934579439252 }, "prompt_5": { "accuracy": 0.5607476635514018 } }, "ph_eval": { "prompt_1": { "accuracy": 0.34, "category_acc": { "brand": 0.3, "demographics": 0.0, "biology": 0.2, "history": 0.13333333333333333, "literature": 0.4, "politics": 0.6, "culture": 0.2, "film": 0.4, "law": 0.4, "geography": 0.7 } }, "prompt_2": { "accuracy": 0.36, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.3, "history": 0.13333333333333333, "literature": 0.4, "politics": 0.6, "culture": 0.2, "film": 0.3, "law": 0.5, "geography": 0.7 } }, "prompt_3": { "accuracy": 0.4, "category_acc": { "brand": 0.6, "demographics": 0.2, "biology": 0.3, "history": 0.13333333333333333, "literature": 0.5, "politics": 0.6, "culture": 0.3, "film": 0.3, "law": 0.5, "geography": 0.6 } }, "prompt_4": { "accuracy": 0.35, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.2, "history": 0.13333333333333333, "literature": 0.3, "politics": 0.6, "culture": 0.3, "film": 0.3, "law": 0.5, "geography": 0.7 } }, "prompt_5": { "accuracy": 0.38, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.3, "history": 0.13333333333333333, "literature": 0.5, "politics": 0.7, "culture": 0.2, "film": 0.3, "law": 0.5, "geography": 0.7 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.19216763243806428 }, "prompt_2": { "bleu_score": 0.17617876480146175 }, "prompt_3": { "bleu_score": 0.1844853053383724 }, "prompt_4": { "bleu_score": 0.19387297841484136 }, "prompt_5": { "bleu_score": 0.14092590287296944 } }, "indommlu": { "prompt_1": { "accuracy": 0.49068696174644505, "category_acc": { "History": 0.4879518072289157, "Geography": 0.46122448979591835, "Lampungic": 0.3197278911564626, "Social science": 0.7646076794657763, "Balinese": 0.29723991507430997, "Makassarese": 0.3172043010752688, "Banjarese": 0.4444444444444444, "Chemistry": 0.29635036496350364, "Biology": 0.44733727810650886, "Science": 0.6212590299277606, "Christian religion": 0.6616915422885572, "Art": 0.6039933444259568, "Islam religion": 0.6002844950213371, "Hindu religion": 0.4666666666666667, "Madurese": 0.31186440677966104, "Sport": 0.5337837837837838, "Indonesian language": 0.5435865504358655, "Physics": 0.4121212121212121, "Minangkabau culture": 0.37185929648241206, "Dayak language": 0.28440366972477066, "Sociology": 0.5020161290322581, "Economy": 0.4672131147540984, "Sundanese": 0.38720829732065687, "Javanese": 0.36794354838709675, "Civic education": 0.6094420600858369 } }, "prompt_2": { "accuracy": 0.4917551238400427, "category_acc": { "History": 0.4959839357429719, "Geography": 0.46938775510204084, "Lampungic": 0.36054421768707484, "Social science": 0.7746243739565943, "Balinese": 0.2951167728237792, "Makassarese": 0.27956989247311825, "Banjarese": 0.4166666666666667, "Chemistry": 0.3021897810218978, "Biology": 0.4544378698224852, "Science": 0.6057791537667698, "Christian religion": 0.6119402985074627, "Art": 0.6372712146422629, "Islam religion": 0.604551920341394, "Hindu religion": 0.49333333333333335, "Madurese": 0.2677966101694915, "Sport": 0.5405405405405406, "Indonesian language": 0.5482565379825654, "Physics": 0.402020202020202, "Minangkabau culture": 0.41708542713567837, "Dayak language": 0.28440366972477066, "Sociology": 0.4717741935483871, "Economy": 0.45286885245901637, "Sundanese": 0.39844425237683667, "Javanese": 0.36794354838709675, "Civic education": 0.6065808297567954 } }, "prompt_3": { "accuracy": 0.5047065892249148, "category_acc": { "History": 0.5261044176706827, "Geography": 0.4816326530612245, "Lampungic": 0.3741496598639456, "Social science": 0.7679465776293823, "Balinese": 0.3205944798301486, "Makassarese": 0.34408602150537637, "Banjarese": 0.4444444444444444, "Chemistry": 0.29635036496350364, "Biology": 0.4544378698224852, "Science": 0.6326109391124871, "Christian religion": 0.6666666666666666, "Art": 0.6156405990016639, "Islam religion": 0.6344238975817923, "Hindu religion": 0.48, "Madurese": 0.2983050847457627, "Sport": 0.5540540540540541, "Indonesian language": 0.5504358655043586, "Physics": 0.4222222222222222, "Minangkabau culture": 0.41708542713567837, "Dayak language": 0.3394495412844037, "Sociology": 0.5080645161290323, "Economy": 0.4733606557377049, "Sundanese": 0.3923941227312014, "Javanese": 0.40524193548387094, "Civic education": 0.6294706723891274 } }, "prompt_4": { "accuracy": 0.5030375859536684, "category_acc": { "History": 0.5220883534136547, "Geography": 0.45714285714285713, "Lampungic": 0.43537414965986393, "Social science": 0.7679465776293823, "Balinese": 0.32908704883227174, "Makassarese": 0.3870967741935484, "Banjarese": 0.4375, "Chemistry": 0.30364963503649633, "Biology": 0.4662721893491124, "Science": 0.6357069143446853, "Christian religion": 0.6567164179104478, "Art": 0.6289517470881864, "Islam religion": 0.6301564722617354, "Hindu religion": 0.46, "Madurese": 0.288135593220339, "Sport": 0.5472972972972973, "Indonesian language": 0.5457658779576587, "Physics": 0.3939393939393939, "Minangkabau culture": 0.41708542713567837, "Dayak language": 0.3486238532110092, "Sociology": 0.49798387096774194, "Economy": 0.48155737704918034, "Sundanese": 0.38202247191011235, "Javanese": 0.39314516129032256, "Civic education": 0.6409155937052933 } }, "prompt_5": { "accuracy": 0.47927097937111957, "category_acc": { "History": 0.4879518072289157, "Geography": 0.42244897959183675, "Lampungic": 0.3401360544217687, "Social science": 0.7395659432387313, "Balinese": 0.28450106157112526, "Makassarese": 0.3763440860215054, "Banjarese": 0.3541666666666667, "Chemistry": 0.2905109489051095, "Biology": 0.4307692307692308, "Science": 0.6130030959752322, "Christian religion": 0.6318407960199005, "Art": 0.5740432612312812, "Islam religion": 0.5889046941678521, "Hindu religion": 0.4266666666666667, "Madurese": 0.3050847457627119, "Sport": 0.5540540540540541, "Indonesian language": 0.5298879202988792, "Physics": 0.3878787878787879, "Minangkabau culture": 0.39195979899497485, "Dayak language": 0.29357798165137616, "Sociology": 0.4415322580645161, "Economy": 0.45081967213114754, "Sundanese": 0.3802938634399309, "Javanese": 0.38306451612903225, "Civic education": 0.6280400572246065 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.33974787333771794 }, "prompt_2": { "bleu_score": 0.3442551398385029 }, "prompt_3": { "bleu_score": 0.33743740022795504 }, "prompt_4": { "bleu_score": 0.33896465094708667 }, "prompt_5": { "bleu_score": 0.3173247794383544 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.2863714706572181 }, "prompt_2": { "bleu_score": 0.2936750253543749 }, "prompt_3": { "bleu_score": 0.2885398011099866 }, "prompt_4": { "bleu_score": 0.29035383506316476 }, "prompt_5": { "bleu_score": 0.28144461734788845 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.20900964720414072 }, "prompt_2": { "bleu_score": 0.21451840379245748 }, "prompt_3": { "bleu_score": 0.20720639696518997 }, "prompt_4": { "bleu_score": 0.20592897490558038 }, "prompt_5": { "bleu_score": 0.20436679638061198 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.3528671298807442 }, "prompt_2": { "bleu_score": 0.3584718766854969 }, "prompt_3": { "bleu_score": 0.3494575250742662 }, "prompt_4": { "bleu_score": 0.35256965248509253 }, "prompt_5": { "bleu_score": 0.3289924681392404 } }, "mmlu": { "prompt_1": { "accuracy": 0.49474912485414235 }, "prompt_2": { "accuracy": 0.5099183197199533 }, "prompt_3": { "accuracy": 0.544924154025671 }, "prompt_4": { "accuracy": 0.5344224037339557 }, "prompt_5": { "accuracy": 0.5344224037339557 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.49717554522702895, "category_acc": { "high_school_european_history": 0.6402439024390244, "business_ethics": 0.6767676767676768, "clinical_knowledge": 0.5075757575757576, "medical_genetics": 0.5959595959595959, "high_school_us_history": 0.5369458128078818, "high_school_physics": 0.34, "high_school_world_history": 0.5805084745762712, "virology": 0.4484848484848485, "high_school_microeconomics": 0.47257383966244726, "econometrics": 0.3893805309734513, "college_computer_science": 0.48484848484848486, "high_school_biology": 0.5922330097087378, "abstract_algebra": 0.36363636363636365, "professional_accounting": 0.3594306049822064, "philosophy": 0.5516129032258065, "professional_medicine": 0.4833948339483395, "nutrition": 0.5245901639344263, "global_facts": 0.36363636363636365, "machine_learning": 0.45045045045045046, "security_studies": 0.4139344262295082, "public_relations": 0.5688073394495413, "professional_psychology": 0.45990180032733224, "prehistory": 0.5510835913312694, "anatomy": 0.48507462686567165, "human_sexuality": 0.5615384615384615, "college_medicine": 0.46511627906976744, "high_school_government_and_politics": 0.6666666666666666, "college_chemistry": 0.41414141414141414, "logical_fallacies": 0.5987654320987654, "high_school_geography": 0.6091370558375635, "elementary_mathematics": 0.44562334217506633, "human_aging": 0.5945945945945946, "college_mathematics": 0.32323232323232326, "high_school_psychology": 0.7077205882352942, "formal_logic": 0.352, "high_school_statistics": 0.386046511627907, "international_law": 0.6583333333333333, "high_school_mathematics": 0.2825278810408922, "high_school_computer_science": 0.5151515151515151, "conceptual_physics": 0.48717948717948717, "miscellaneous": 0.7429667519181585, "high_school_chemistry": 0.3910891089108911, "marketing": 0.6909871244635193, "professional_law": 0.3568166992824527, "management": 0.6274509803921569, "college_physics": 0.32673267326732675, "jurisprudence": 0.5046728971962616, "world_religions": 0.7, "sociology": 0.725, "us_foreign_policy": 0.6565656565656566, "high_school_macroeconomics": 0.4961439588688946, "computer_security": 0.6464646464646465, "moral_scenarios": 0.2751677852348993, "moral_disputes": 0.527536231884058, "electrical_engineering": 0.4861111111111111, "astronomy": 0.5099337748344371, "college_biology": 0.5244755244755245 } }, "prompt_2": { "accuracy": 0.5031104755094744, "category_acc": { "high_school_european_history": 0.676829268292683, "business_ethics": 0.6868686868686869, "clinical_knowledge": 0.553030303030303, "medical_genetics": 0.6060606060606061, "high_school_us_history": 0.5566502463054187, "high_school_physics": 0.36, "high_school_world_history": 0.614406779661017, "virology": 0.4121212121212121, "high_school_microeconomics": 0.48945147679324896, "econometrics": 0.35398230088495575, "college_computer_science": 0.47474747474747475, "high_school_biology": 0.6019417475728155, "abstract_algebra": 0.31313131313131315, "professional_accounting": 0.3914590747330961, "philosophy": 0.5709677419354838, "professional_medicine": 0.4907749077490775, "nutrition": 0.5377049180327869, "global_facts": 0.1919191919191919, "machine_learning": 0.43243243243243246, "security_studies": 0.5163934426229508, "public_relations": 0.5045871559633027, "professional_psychology": 0.48117839607201307, "prehistory": 0.4953560371517028, "anatomy": 0.48507462686567165, "human_sexuality": 0.5461538461538461, "college_medicine": 0.47093023255813954, "high_school_government_and_politics": 0.734375, "college_chemistry": 0.37373737373737376, "logical_fallacies": 0.5925925925925926, "high_school_geography": 0.6751269035532995, "elementary_mathematics": 0.4403183023872679, "human_aging": 0.581081081081081, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.7352941176470589, "formal_logic": 0.384, "high_school_statistics": 0.4, "international_law": 0.6416666666666667, "high_school_mathematics": 0.26765799256505574, "high_school_computer_science": 0.5353535353535354, "conceptual_physics": 0.49572649572649574, "miscellaneous": 0.7468030690537084, "high_school_chemistry": 0.4603960396039604, "marketing": 0.6824034334763949, "professional_law": 0.35094585779517284, "management": 0.6274509803921569, "college_physics": 0.33663366336633666, "jurisprudence": 0.5046728971962616, "world_religions": 0.7176470588235294, "sociology": 0.715, "us_foreign_policy": 0.696969696969697, "high_school_macroeconomics": 0.5115681233933161, "computer_security": 0.6464646464646465, "moral_scenarios": 0.2684563758389262, "moral_disputes": 0.5391304347826087, "electrical_engineering": 0.4791666666666667, "astronomy": 0.5165562913907285, "college_biology": 0.48951048951048953 } }, "prompt_3": { "accuracy": 0.5221308544869503, "category_acc": { "high_school_european_history": 0.6158536585365854, "business_ethics": 0.6666666666666666, "clinical_knowledge": 0.5833333333333334, "medical_genetics": 0.6464646464646465, "high_school_us_history": 0.6206896551724138, "high_school_physics": 0.34, "high_school_world_history": 0.6864406779661016, "virology": 0.42424242424242425, "high_school_microeconomics": 0.5147679324894515, "econometrics": 0.34513274336283184, "college_computer_science": 0.43434343434343436, "high_school_biology": 0.6375404530744336, "abstract_algebra": 0.31313131313131315, "professional_accounting": 0.39501779359430605, "philosophy": 0.6064516129032258, "professional_medicine": 0.4870848708487085, "nutrition": 0.5475409836065573, "global_facts": 0.29292929292929293, "machine_learning": 0.45045045045045046, "security_studies": 0.5614754098360656, "public_relations": 0.5779816513761468, "professional_psychology": 0.486088379705401, "prehistory": 0.5851393188854489, "anatomy": 0.44029850746268656, "human_sexuality": 0.5923076923076923, "college_medicine": 0.4941860465116279, "high_school_government_and_politics": 0.75, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.6172839506172839, "high_school_geography": 0.7055837563451777, "elementary_mathematics": 0.4509283819628647, "human_aging": 0.6126126126126126, "college_mathematics": 0.29292929292929293, "high_school_psychology": 0.7610294117647058, "formal_logic": 0.408, "high_school_statistics": 0.4046511627906977, "international_law": 0.6416666666666667, "high_school_mathematics": 0.2825278810408922, "high_school_computer_science": 0.5252525252525253, "conceptual_physics": 0.5128205128205128, "miscellaneous": 0.7583120204603581, "high_school_chemistry": 0.41089108910891087, "marketing": 0.7682403433476395, "professional_law": 0.3724722765818656, "management": 0.6470588235294118, "college_physics": 0.27722772277227725, "jurisprudence": 0.5700934579439252, "world_religions": 0.7294117647058823, "sociology": 0.74, "us_foreign_policy": 0.7070707070707071, "high_school_macroeconomics": 0.5295629820051414, "computer_security": 0.696969696969697, "moral_scenarios": 0.25727069351230425, "moral_disputes": 0.6202898550724638, "electrical_engineering": 0.4791666666666667, "astronomy": 0.5165562913907285, "college_biology": 0.5034965034965035 } }, "prompt_4": { "accuracy": 0.5174830175187701, "category_acc": { "high_school_european_history": 0.6585365853658537, "business_ethics": 0.6565656565656566, "clinical_knowledge": 0.5416666666666666, "medical_genetics": 0.6161616161616161, "high_school_us_history": 0.6009852216748769, "high_school_physics": 0.32, "high_school_world_history": 0.6398305084745762, "virology": 0.44242424242424244, "high_school_microeconomics": 0.5147679324894515, "econometrics": 0.37168141592920356, "college_computer_science": 0.45454545454545453, "high_school_biology": 0.6148867313915858, "abstract_algebra": 0.35353535353535354, "professional_accounting": 0.4128113879003559, "philosophy": 0.5870967741935483, "professional_medicine": 0.4833948339483395, "nutrition": 0.5737704918032787, "global_facts": 0.32323232323232326, "machine_learning": 0.45045045045045046, "security_studies": 0.5573770491803278, "public_relations": 0.5871559633027523, "professional_psychology": 0.4877250409165303, "prehistory": 0.5727554179566563, "anatomy": 0.4701492537313433, "human_sexuality": 0.6076923076923076, "college_medicine": 0.4941860465116279, "high_school_government_and_politics": 0.7239583333333334, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.6296296296296297, "high_school_geography": 0.6598984771573604, "elementary_mathematics": 0.40583554376657827, "human_aging": 0.6171171171171171, "college_mathematics": 0.29292929292929293, "high_school_psychology": 0.7316176470588235, "formal_logic": 0.384, "high_school_statistics": 0.42790697674418604, "international_law": 0.6416666666666667, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.5555555555555556, "conceptual_physics": 0.49145299145299143, "miscellaneous": 0.7480818414322251, "high_school_chemistry": 0.45544554455445546, "marketing": 0.7381974248927039, "professional_law": 0.3724722765818656, "management": 0.6470588235294118, "college_physics": 0.297029702970297, "jurisprudence": 0.6074766355140186, "world_religions": 0.7235294117647059, "sociology": 0.75, "us_foreign_policy": 0.7070707070707071, "high_school_macroeconomics": 0.5038560411311054, "computer_security": 0.6767676767676768, "moral_scenarios": 0.27628635346756153, "moral_disputes": 0.5797101449275363, "electrical_engineering": 0.4791666666666667, "astronomy": 0.48344370860927155, "college_biology": 0.5244755244755245 } }, "prompt_5": { "accuracy": 0.5158383982838756, "category_acc": { "high_school_european_history": 0.6402439024390244, "business_ethics": 0.6363636363636364, "clinical_knowledge": 0.5643939393939394, "medical_genetics": 0.6464646464646465, "high_school_us_history": 0.625615763546798, "high_school_physics": 0.32, "high_school_world_history": 0.635593220338983, "virology": 0.4121212121212121, "high_school_microeconomics": 0.5063291139240507, "econometrics": 0.35398230088495575, "college_computer_science": 0.41414141414141414, "high_school_biology": 0.6116504854368932, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.4092526690391459, "philosophy": 0.5870967741935483, "professional_medicine": 0.4907749077490775, "nutrition": 0.5672131147540984, "global_facts": 0.3333333333333333, "machine_learning": 0.4864864864864865, "security_studies": 0.5655737704918032, "public_relations": 0.5779816513761468, "professional_psychology": 0.4762684124386252, "prehistory": 0.56656346749226, "anatomy": 0.48507462686567165, "human_sexuality": 0.6230769230769231, "college_medicine": 0.5348837209302325, "high_school_government_and_politics": 0.6927083333333334, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.6172839506172839, "high_school_geography": 0.6903553299492385, "elementary_mathematics": 0.44297082228116713, "human_aging": 0.6261261261261262, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.7261029411764706, "formal_logic": 0.4, "high_school_statistics": 0.40930232558139534, "international_law": 0.6583333333333333, "high_school_mathematics": 0.26022304832713755, "high_school_computer_science": 0.5252525252525253, "conceptual_physics": 0.46153846153846156, "miscellaneous": 0.7442455242966752, "high_school_chemistry": 0.4405940594059406, "marketing": 0.7553648068669528, "professional_law": 0.36268754076973253, "management": 0.6568627450980392, "college_physics": 0.25742574257425743, "jurisprudence": 0.5887850467289719, "world_religions": 0.7, "sociology": 0.735, "us_foreign_policy": 0.7171717171717171, "high_school_macroeconomics": 0.5115681233933161, "computer_security": 0.6666666666666666, "moral_scenarios": 0.28747203579418346, "moral_disputes": 0.5768115942028985, "electrical_engineering": 0.4930555555555556, "astronomy": 0.5033112582781457, "college_biology": 0.5314685314685315 } } }, "c_eval": { "prompt_1": { "accuracy": 0.5505200594353641 }, "prompt_2": { "accuracy": 0.5586924219910847 }, "prompt_3": { "accuracy": 0.5505200594353641 }, "prompt_4": { "accuracy": 0.5416047548291233 }, "prompt_5": { "accuracy": 0.537890044576523 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.5510585305105853, "category_acc": { "computer_network": 0.5, "operating_system": 0.5416666666666666, "computer_architecture": 0.5384615384615384, "college_programming": 0.47619047619047616, "college_physics": 0.4583333333333333, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.09523809523809523, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.5862068965517241, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.5416666666666666, "high_school_biology": 0.5, "middle_school_mathematics": 0.4166666666666667, "middle_school_biology": 0.8076923076923077, "middle_school_physics": 0.7916666666666666, "middle_school_chemistry": 0.88, "veterinary_medicine": 0.4642857142857143, "college_economics": 0.43333333333333335, "business_administration": 0.47368421052631576, "marxism": 0.75, "mao_zedong_thought": 0.7241379310344828, "education_science": 0.7647058823529411, "teacher_qualification": 0.7755102040816326, "high_school_politics": 0.875, "high_school_geography": 0.7083333333333334, "middle_school_politics": 0.8076923076923077, "middle_school_geography": 0.8823529411764706, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.875, "logic": 0.5185185185185185, "law": 0.3448275862068966, "chinese_language_and_literature": 0.5357142857142857, "art_studies": 0.5263157894736842, "professional_tour_guide": 0.6470588235294118, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.5416666666666666, "high_school_history": 0.8, "middle_school_history": 0.7777777777777778, "civil_servant": 0.4230769230769231, "sports_science": 0.7083333333333334, "plant_protection": 0.5925925925925926, "basic_medicine": 0.5, "clinical_medicine": 0.5555555555555556, "urban_and_rural_planner": 0.5098039215686274, "accountant": 0.4444444444444444, "fire_engineer": 0.4444444444444444, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.48148148148148145, "physician": 0.5370370370370371 } }, "prompt_2": { "accuracy": 0.5566625155666252, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.5416666666666666, "computer_architecture": 0.46153846153846156, "college_programming": 0.47619047619047616, "college_physics": 0.375, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.375, "high_school_chemistry": 0.5416666666666666, "high_school_biology": 0.5416666666666666, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.8076923076923077, "middle_school_physics": 0.8333333333333334, "middle_school_chemistry": 0.92, "veterinary_medicine": 0.5, "college_economics": 0.4666666666666667, "business_administration": 0.47368421052631576, "marxism": 0.875, "mao_zedong_thought": 0.7241379310344828, "education_science": 0.7058823529411765, "teacher_qualification": 0.7551020408163265, "high_school_politics": 0.8333333333333334, "high_school_geography": 0.75, "middle_school_politics": 0.9230769230769231, "middle_school_geography": 0.7647058823529411, "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.7916666666666666, "logic": 0.48148148148148145, "law": 0.4482758620689655, "chinese_language_and_literature": 0.6071428571428571, "art_studies": 0.5263157894736842, "professional_tour_guide": 0.6470588235294118, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.6666666666666666, "high_school_history": 0.76, "middle_school_history": 0.7777777777777778, "civil_servant": 0.4423076923076923, "sports_science": 0.5833333333333334, "plant_protection": 0.7037037037037037, "basic_medicine": 0.5416666666666666, "clinical_medicine": 0.5185185185185185, "urban_and_rural_planner": 0.5098039215686274, "accountant": 0.4444444444444444, "fire_engineer": 0.4166666666666667, "environmental_impact_assessment_engineer": 0.5833333333333334, "tax_accountant": 0.48148148148148145, "physician": 0.5370370370370371 } }, "prompt_3": { "accuracy": 0.5554171855541719, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5416666666666666, "computer_architecture": 0.46153846153846156, "college_programming": 0.47619047619047616, "college_physics": 0.4166666666666667, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.375, "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.4523809523809524, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.4166666666666667, "high_school_chemistry": 0.5, "high_school_biology": 0.5416666666666666, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.8076923076923077, "middle_school_physics": 0.8333333333333334, "middle_school_chemistry": 0.88, "veterinary_medicine": 0.5, "college_economics": 0.4166666666666667, "business_administration": 0.5, "marxism": 0.875, "mao_zedong_thought": 0.6896551724137931, "education_science": 0.7352941176470589, "teacher_qualification": 0.7346938775510204, "high_school_politics": 0.8333333333333334, "high_school_geography": 0.7083333333333334, "middle_school_politics": 0.8846153846153846, "middle_school_geography": 0.7647058823529411, "modern_chinese_history": 0.5357142857142857, "ideological_and_moral_cultivation": 0.7916666666666666, "logic": 0.4444444444444444, "law": 0.4482758620689655, "chinese_language_and_literature": 0.5714285714285714, "art_studies": 0.5263157894736842, "professional_tour_guide": 0.5882352941176471, "legal_professional": 0.5714285714285714, "high_school_chinese": 0.625, "high_school_history": 0.68, "middle_school_history": 0.7777777777777778, "civil_servant": 0.46153846153846156, "sports_science": 0.5833333333333334, "plant_protection": 0.6666666666666666, "basic_medicine": 0.5833333333333334, "clinical_medicine": 0.48148148148148145, "urban_and_rural_planner": 0.5490196078431373, "accountant": 0.46296296296296297, "fire_engineer": 0.4444444444444444, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.42592592592592593, "physician": 0.5740740740740741 } }, "prompt_4": { "accuracy": 0.5354919053549191, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.4583333333333333, "computer_architecture": 0.5384615384615384, "college_programming": 0.42857142857142855, "college_physics": 0.4583333333333333, "college_chemistry": 0.1724137931034483, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.4782608695652174, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.5416666666666666, "high_school_chemistry": 0.5, "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.8076923076923077, "middle_school_physics": 0.8333333333333334, "middle_school_chemistry": 0.76, "veterinary_medicine": 0.4642857142857143, "college_economics": 0.5, "business_administration": 0.5, "marxism": 0.875, "mao_zedong_thought": 0.7241379310344828, "education_science": 0.7058823529411765, "teacher_qualification": 0.7755102040816326, "high_school_politics": 0.8333333333333334, "high_school_geography": 0.7083333333333334, "middle_school_politics": 0.9230769230769231, "middle_school_geography": 0.7647058823529411, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.875, "logic": 0.4444444444444444, "law": 0.41379310344827586, "chinese_language_and_literature": 0.5, "art_studies": 0.5526315789473685, "professional_tour_guide": 0.6470588235294118, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.4583333333333333, "high_school_history": 0.76, "middle_school_history": 0.7777777777777778, "civil_servant": 0.36538461538461536, "sports_science": 0.5833333333333334, "plant_protection": 0.5555555555555556, "basic_medicine": 0.5416666666666666, "clinical_medicine": 0.5185185185185185, "urban_and_rural_planner": 0.47058823529411764, "accountant": 0.42592592592592593, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.5833333333333334, "tax_accountant": 0.4074074074074074, "physician": 0.5185185185185185 } }, "prompt_5": { "accuracy": 0.5473225404732254, "category_acc": { "computer_network": 0.5, "operating_system": 0.4583333333333333, "computer_architecture": 0.5, "college_programming": 0.5, "college_physics": 0.3333333333333333, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.375, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.38095238095238093, "metrology_engineer": 0.5862068965517241, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.5, "high_school_chemistry": 0.5, "high_school_biology": 0.5416666666666666, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.7307692307692307, "middle_school_physics": 0.75, "middle_school_chemistry": 0.8, "veterinary_medicine": 0.5, "college_economics": 0.45, "business_administration": 0.5526315789473685, "marxism": 0.8333333333333334, "mao_zedong_thought": 0.7586206896551724, "education_science": 0.6764705882352942, "teacher_qualification": 0.7551020408163265, "high_school_politics": 0.9166666666666666, "high_school_geography": 0.6666666666666666, "middle_school_politics": 0.9230769230769231, "middle_school_geography": 0.7647058823529411, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.8333333333333334, "logic": 0.4444444444444444, "law": 0.3448275862068966, "chinese_language_and_literature": 0.5357142857142857, "art_studies": 0.5263157894736842, "professional_tour_guide": 0.6470588235294118, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.6666666666666666, "high_school_history": 0.84, "middle_school_history": 0.8148148148148148, "civil_servant": 0.46153846153846156, "sports_science": 0.6666666666666666, "plant_protection": 0.5555555555555556, "basic_medicine": 0.5833333333333334, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.5882352941176471, "accountant": 0.42592592592592593, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.4074074074074074, "physician": 0.5555555555555556 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.5483870967741935 }, "prompt_2": { "accuracy": 0.5698924731182796 }, "prompt_3": { "accuracy": 0.5412186379928315 }, "prompt_4": { "accuracy": 0.5663082437275986 }, "prompt_5": { "accuracy": 0.5627240143369175 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.5473147988257642, "category_acc": { "agronomy": 0.5266272189349113, "anatomy": 0.44594594594594594, "ancient_chinese": 0.35365853658536583, "arts": 0.7625, "astronomy": 0.3393939393939394, "business_ethics": 0.5789473684210527, "chinese_civil_service_exam": 0.55, "chinese_driving_rule": 0.7709923664122137, "chinese_food_culture": 0.5441176470588235, "chinese_foreign_policy": 0.6074766355140186, "chinese_history": 0.6749226006191951, "chinese_literature": 0.43137254901960786, "chinese_teacher_qualification": 0.7318435754189944, "clinical_knowledge": 0.4936708860759494, "college_actuarial_science": 0.27358490566037735, "college_education": 0.6542056074766355, "college_engineering_hydrology": 0.5094339622641509, "college_law": 0.46296296296296297, "college_mathematics": 0.34285714285714286, "college_medical_statistics": 0.39622641509433965, "college_medicine": 0.5018315018315018, "computer_science": 0.5588235294117647, "computer_security": 0.6842105263157895, "conceptual_physics": 0.6462585034013606, "construction_project_management": 0.4460431654676259, "economics": 0.5786163522012578, "education": 0.656441717791411, "electrical_engineering": 0.4941860465116279, "elementary_chinese": 0.5317460317460317, "elementary_commonsense": 0.5656565656565656, "elementary_information_and_technology": 0.7310924369747899, "elementary_mathematics": 0.32608695652173914, "ethnology": 0.5851851851851851, "food_science": 0.5454545454545454, "genetics": 0.44886363636363635, "global_facts": 0.5838926174496645, "high_school_biology": 0.5088757396449705, "high_school_chemistry": 0.5075757575757576, "high_school_geography": 0.5677966101694916, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.4090909090909091, "high_school_politics": 0.5524475524475524, "human_sexuality": 0.5476190476190477, "international_law": 0.4648648648648649, "journalism": 0.47093023255813954, "jurisprudence": 0.537712895377129, "legal_and_moral_basis": 0.8598130841121495, "logical": 0.4715447154471545, "machine_learning": 0.4672131147540984, "management": 0.6619047619047619, "marketing": 0.6333333333333333, "marxist_theory": 0.6613756613756614, "modern_chinese": 0.39655172413793105, "nutrition": 0.593103448275862, "philosophy": 0.638095238095238, "professional_accounting": 0.6, "professional_law": 0.5071090047393365, "professional_medicine": 0.40425531914893614, "professional_psychology": 0.6293103448275862, "public_relations": 0.5977011494252874, "security_study": 0.6666666666666666, "sociology": 0.5796460176991151, "sports_science": 0.5151515151515151, "traditional_chinese_medicine": 0.5243243243243243, "virology": 0.5325443786982249, "world_history": 0.6335403726708074, "world_religions": 0.65625 } }, "prompt_2": { "accuracy": 0.5558625453289587, "category_acc": { "agronomy": 0.5207100591715976, "anatomy": 0.43243243243243246, "ancient_chinese": 0.3231707317073171, "arts": 0.76875, "astronomy": 0.4, "business_ethics": 0.5741626794258373, "chinese_civil_service_exam": 0.55, "chinese_driving_rule": 0.8015267175572519, "chinese_food_culture": 0.5735294117647058, "chinese_foreign_policy": 0.6074766355140186, "chinese_history": 0.6811145510835913, "chinese_literature": 0.4215686274509804, "chinese_teacher_qualification": 0.7374301675977654, "clinical_knowledge": 0.5147679324894515, "college_actuarial_science": 0.36792452830188677, "college_education": 0.6635514018691588, "college_engineering_hydrology": 0.5188679245283019, "college_law": 0.49074074074074076, "college_mathematics": 0.3333333333333333, "college_medical_statistics": 0.4056603773584906, "college_medicine": 0.5201465201465202, "computer_science": 0.5931372549019608, "computer_security": 0.7017543859649122, "conceptual_physics": 0.6190476190476191, "construction_project_management": 0.45323741007194246, "economics": 0.5911949685534591, "education": 0.6073619631901841, "electrical_engineering": 0.48255813953488375, "elementary_chinese": 0.5515873015873016, "elementary_commonsense": 0.5707070707070707, "elementary_information_and_technology": 0.7310924369747899, "elementary_mathematics": 0.34782608695652173, "ethnology": 0.5481481481481482, "food_science": 0.5314685314685315, "genetics": 0.4318181818181818, "global_facts": 0.5906040268456376, "high_school_biology": 0.514792899408284, "high_school_chemistry": 0.5151515151515151, "high_school_geography": 0.576271186440678, "high_school_mathematics": 0.29878048780487804, "high_school_physics": 0.4636363636363636, "high_school_politics": 0.5454545454545454, "human_sexuality": 0.5238095238095238, "international_law": 0.4810810810810811, "journalism": 0.5406976744186046, "jurisprudence": 0.5401459854014599, "legal_and_moral_basis": 0.8504672897196262, "logical": 0.4715447154471545, "machine_learning": 0.47540983606557374, "management": 0.6761904761904762, "marketing": 0.65, "marxist_theory": 0.6666666666666666, "modern_chinese": 0.4224137931034483, "nutrition": 0.6344827586206897, "philosophy": 0.6476190476190476, "professional_accounting": 0.6057142857142858, "professional_law": 0.46445497630331756, "professional_medicine": 0.425531914893617, "professional_psychology": 0.6508620689655172, "public_relations": 0.5804597701149425, "security_study": 0.674074074074074, "sociology": 0.5752212389380531, "sports_science": 0.5454545454545454, "traditional_chinese_medicine": 0.5513513513513514, "virology": 0.5502958579881657, "world_history": 0.6770186335403726, "world_religions": 0.65625 } }, "prompt_3": { "accuracy": 0.5527542738732516, "category_acc": { "agronomy": 0.5443786982248521, "anatomy": 0.44594594594594594, "ancient_chinese": 0.31097560975609756, "arts": 0.75625, "astronomy": 0.4, "business_ethics": 0.569377990430622, "chinese_civil_service_exam": 0.5375, "chinese_driving_rule": 0.7480916030534351, "chinese_food_culture": 0.5441176470588235, "chinese_foreign_policy": 0.6074766355140186, "chinese_history": 0.6842105263157895, "chinese_literature": 0.4264705882352941, "chinese_teacher_qualification": 0.7430167597765364, "clinical_knowledge": 0.5021097046413502, "college_actuarial_science": 0.29245283018867924, "college_education": 0.6355140186915887, "college_engineering_hydrology": 0.5, "college_law": 0.4537037037037037, "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.44339622641509435, "college_medicine": 0.5054945054945055, "computer_science": 0.5637254901960784, "computer_security": 0.7017543859649122, "conceptual_physics": 0.6258503401360545, "construction_project_management": 0.4748201438848921, "economics": 0.5911949685534591, "education": 0.6134969325153374, "electrical_engineering": 0.5232558139534884, "elementary_chinese": 0.5476190476190477, "elementary_commonsense": 0.5555555555555556, "elementary_information_and_technology": 0.7352941176470589, "elementary_mathematics": 0.3826086956521739, "ethnology": 0.5925925925925926, "food_science": 0.5944055944055944, "genetics": 0.42613636363636365, "global_facts": 0.5838926174496645, "high_school_biology": 0.4970414201183432, "high_school_chemistry": 0.553030303030303, "high_school_geography": 0.4915254237288136, "high_school_mathematics": 0.32926829268292684, "high_school_physics": 0.41818181818181815, "high_school_politics": 0.5874125874125874, "human_sexuality": 0.5555555555555556, "international_law": 0.4702702702702703, "journalism": 0.5406976744186046, "jurisprudence": 0.5547445255474452, "legal_and_moral_basis": 0.8504672897196262, "logical": 0.4878048780487805, "machine_learning": 0.45901639344262296, "management": 0.6761904761904762, "marketing": 0.6555555555555556, "marxist_theory": 0.656084656084656, "modern_chinese": 0.4396551724137931, "nutrition": 0.6275862068965518, "philosophy": 0.5904761904761905, "professional_accounting": 0.6114285714285714, "professional_law": 0.44549763033175355, "professional_medicine": 0.3882978723404255, "professional_psychology": 0.6551724137931034, "public_relations": 0.5862068965517241, "security_study": 0.6814814814814815, "sociology": 0.5663716814159292, "sports_science": 0.5212121212121212, "traditional_chinese_medicine": 0.5459459459459459, "virology": 0.5443786982248521, "world_history": 0.6708074534161491, "world_religions": 0.63125 } }, "prompt_4": { "accuracy": 0.5395441201864963, "category_acc": { "agronomy": 0.5088757396449705, "anatomy": 0.44594594594594594, "ancient_chinese": 0.32926829268292684, "arts": 0.75, "astronomy": 0.3696969696969697, "business_ethics": 0.569377990430622, "chinese_civil_service_exam": 0.5375, "chinese_driving_rule": 0.7480916030534351, "chinese_food_culture": 0.5514705882352942, "chinese_foreign_policy": 0.5887850467289719, "chinese_history": 0.6501547987616099, "chinese_literature": 0.45098039215686275, "chinese_teacher_qualification": 0.7430167597765364, "clinical_knowledge": 0.4978902953586498, "college_actuarial_science": 0.36792452830188677, "college_education": 0.6542056074766355, "college_engineering_hydrology": 0.5283018867924528, "college_law": 0.46296296296296297, "college_mathematics": 0.3142857142857143, "college_medical_statistics": 0.37735849056603776, "college_medicine": 0.5054945054945055, "computer_science": 0.5637254901960784, "computer_security": 0.6666666666666666, "conceptual_physics": 0.6326530612244898, "construction_project_management": 0.4460431654676259, "economics": 0.5660377358490566, "education": 0.6012269938650306, "electrical_engineering": 0.5174418604651163, "elementary_chinese": 0.5317460317460317, "elementary_commonsense": 0.5606060606060606, "elementary_information_and_technology": 0.7352941176470589, "elementary_mathematics": 0.3695652173913043, "ethnology": 0.5481481481481482, "food_science": 0.6083916083916084, "genetics": 0.4431818181818182, "global_facts": 0.5906040268456376, "high_school_biology": 0.44970414201183434, "high_school_chemistry": 0.4696969696969697, "high_school_geography": 0.5254237288135594, "high_school_mathematics": 0.2682926829268293, "high_school_physics": 0.42727272727272725, "high_school_politics": 0.5244755244755245, "human_sexuality": 0.5079365079365079, "international_law": 0.44324324324324327, "journalism": 0.5116279069767442, "jurisprudence": 0.5304136253041363, "legal_and_moral_basis": 0.8411214953271028, "logical": 0.4715447154471545, "machine_learning": 0.45901639344262296, "management": 0.6476190476190476, "marketing": 0.6277777777777778, "marxist_theory": 0.6190476190476191, "modern_chinese": 0.3706896551724138, "nutrition": 0.5586206896551724, "philosophy": 0.5904761904761905, "professional_accounting": 0.5828571428571429, "professional_law": 0.46919431279620855, "professional_medicine": 0.39361702127659576, "professional_psychology": 0.6206896551724138, "public_relations": 0.5632183908045977, "security_study": 0.6592592592592592, "sociology": 0.5530973451327433, "sports_science": 0.5272727272727272, "traditional_chinese_medicine": 0.5297297297297298, "virology": 0.5325443786982249, "world_history": 0.6211180124223602, "world_religions": 0.65625 } }, "prompt_5": { "accuracy": 0.5513728198929373, "category_acc": { "agronomy": 0.514792899408284, "anatomy": 0.5, "ancient_chinese": 0.3353658536585366, "arts": 0.7625, "astronomy": 0.3212121212121212, "business_ethics": 0.6172248803827751, "chinese_civil_service_exam": 0.55625, "chinese_driving_rule": 0.7633587786259542, "chinese_food_culture": 0.5147058823529411, "chinese_foreign_policy": 0.6261682242990654, "chinese_history": 0.6780185758513931, "chinese_literature": 0.45098039215686275, "chinese_teacher_qualification": 0.7430167597765364, "clinical_knowledge": 0.510548523206751, "college_actuarial_science": 0.3018867924528302, "college_education": 0.6635514018691588, "college_engineering_hydrology": 0.49056603773584906, "college_law": 0.4722222222222222, "college_mathematics": 0.3238095238095238, "college_medical_statistics": 0.41509433962264153, "college_medicine": 0.5054945054945055, "computer_science": 0.5637254901960784, "computer_security": 0.7134502923976608, "conceptual_physics": 0.6258503401360545, "construction_project_management": 0.4676258992805755, "economics": 0.5786163522012578, "education": 0.6073619631901841, "electrical_engineering": 0.5465116279069767, "elementary_chinese": 0.5396825396825397, "elementary_commonsense": 0.5656565656565656, "elementary_information_and_technology": 0.7563025210084033, "elementary_mathematics": 0.3695652173913043, "ethnology": 0.5777777777777777, "food_science": 0.5454545454545454, "genetics": 0.4034090909090909, "global_facts": 0.5436241610738255, "high_school_biology": 0.4437869822485207, "high_school_chemistry": 0.5075757575757576, "high_school_geography": 0.5508474576271186, "high_school_mathematics": 0.25, "high_school_physics": 0.4636363636363636, "high_school_politics": 0.5804195804195804, "human_sexuality": 0.5476190476190477, "international_law": 0.43783783783783786, "journalism": 0.5232558139534884, "jurisprudence": 0.5669099756690997, "legal_and_moral_basis": 0.8271028037383178, "logical": 0.4878048780487805, "machine_learning": 0.4426229508196721, "management": 0.6619047619047619, "marketing": 0.6222222222222222, "marxist_theory": 0.6455026455026455, "modern_chinese": 0.4051724137931034, "nutrition": 0.5310344827586206, "philosophy": 0.6190476190476191, "professional_accounting": 0.5771428571428572, "professional_law": 0.4976303317535545, "professional_medicine": 0.425531914893617, "professional_psychology": 0.6379310344827587, "public_relations": 0.6149425287356322, "security_study": 0.6814814814814815, "sociology": 0.584070796460177, "sports_science": 0.5212121212121212, "traditional_chinese_medicine": 0.5621621621621622, "virology": 0.5680473372781065, "world_history": 0.6521739130434783, "world_religions": 0.68125 } } }, "zbench": { "prompt_1": { "accuracy": 0.36363636363636365 }, "prompt_2": { "accuracy": 0.30303030303030304 }, "prompt_3": { "accuracy": 0.36363636363636365 }, "prompt_4": { "accuracy": 0.3333333333333333 }, "prompt_5": { "accuracy": 0.36363636363636365 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.5590909090909091 }, "prompt_3": { "accuracy": 0.6090909090909091 }, "prompt_4": { "accuracy": 0.6204545454545455 }, "prompt_5": { "accuracy": 0.6 } }, "ocnli": { "prompt_1": { "accuracy": 0.49457627118644065 }, "prompt_2": { "accuracy": 0.4847457627118644 }, "prompt_3": { "accuracy": 0.4942372881355932 }, "prompt_4": { "accuracy": 0.4603389830508475 }, "prompt_5": { "accuracy": 0.4935593220338983 } }, "c3": { "prompt_1": { "accuracy": 0.8534031413612565 }, "prompt_2": { "accuracy": 0.8590127150336574 }, "prompt_3": { "accuracy": 0.8548990276738968 }, "prompt_4": { "accuracy": 0.8515332834704562 }, "prompt_5": { "accuracy": 0.8526551982049364 } }, "dream": { "prompt_1": { "accuracy": 0.7133757961783439 }, "prompt_2": { "accuracy": 0.7506124448799608 }, "prompt_3": { "accuracy": 0.7261146496815286 }, "prompt_4": { "accuracy": 0.7275845173934345 }, "prompt_5": { "accuracy": 0.7207251347378736 } }, "samsum": { "prompt_1": { "rouge1": 0.3380038032957861, "rouge2": 0.11233090757630733, "rougeL": 0.2525846961222128, "avg_rouge": 0.23430646899810206 }, "prompt_2": { "rouge1": 0.3402688720384836, "rouge2": 0.11617170717611897, "rougeL": 0.2556909208109999, "avg_rouge": 0.23737716667520084 }, "prompt_3": { "rouge1": 0.35698629071621746, "rouge2": 0.11854058162244702, "rougeL": 0.2686418609896302, "avg_rouge": 0.2480562444427649 }, "prompt_4": { "rouge1": 0.3342994800625261, "rouge2": 0.11240749315987966, "rougeL": 0.2490189475006745, "avg_rouge": 0.23190864024102673 }, "prompt_5": { "rouge1": 0.33860545114927265, "rouge2": 0.11058705018769807, "rougeL": 0.25443600103582137, "avg_rouge": 0.23454283412426405 } }, "dialogsum": { "prompt_1": { "rouge1": 0.21397562710108725, "rouge2": 0.057891940912959426, "rougeL": 0.15740443114264127, "avg_rouge": 0.14309066638556264 }, "prompt_2": { "rouge1": 0.21161044567205908, "rouge2": 0.054305946671291946, "rougeL": 0.15502329163680859, "avg_rouge": 0.14031322799338652 }, "prompt_3": { "rouge1": 0.212214415398881, "rouge2": 0.054087532294718216, "rougeL": 0.1553242277622777, "avg_rouge": 0.1405420584852923 }, "prompt_4": { "rouge1": 0.21398263404053994, "rouge2": 0.057143774377134346, "rougeL": 0.15730363685762183, "avg_rouge": 0.14281001509176536 }, "prompt_5": { "rouge1": 0.2109018158847188, "rouge2": 0.05370712413153861, "rougeL": 0.15883621981098608, "avg_rouge": 0.14114838660908116 } }, "sst2": { "prompt_1": { "accuracy": 0.9128440366972477 }, "prompt_2": { "accuracy": 0.911697247706422 }, "prompt_3": { "accuracy": 0.8727064220183486 }, "prompt_4": { "accuracy": 0.9139908256880734 }, "prompt_5": { "accuracy": 0.8887614678899083 } }, "cola": { "prompt_1": { "accuracy": 0.7075743048897412 }, "prompt_2": { "accuracy": 0.725790987535954 }, "prompt_3": { "accuracy": 0.7181208053691275 }, "prompt_4": { "accuracy": 0.7488015340364333 }, "prompt_5": { "accuracy": 0.7660594439117929 } }, "qqp": { "prompt_1": { "accuracy": 0.778 }, "prompt_2": { "accuracy": 0.794 }, "prompt_3": { "accuracy": 0.779 }, "prompt_4": { "accuracy": 0.742 }, "prompt_5": { "accuracy": 0.7955 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.6 } }, "qnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } }, "rte": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.9 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "sea_mistral_highest_acc_inst_7b": { "model_size": "7B", "model_link": "https://seaeval.github.io/", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.8836134453781513, "language_acc": { "Spanish": 0.9, "English": 0.9235294117647059, "Chinese": 0.8705882352941177, "Vietnamese": 0.8403361344537815 }, "consistency_score_2": 0.8586834733893557, "consistency_score_3": 0.7983193277310924, "consistency_score_4": 0.7605042016806722, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9201680672268907, "Spanish,Chinese": 0.8596638655462185, "Spanish,Vietnamese": 0.8436974789915966, "English,Chinese": 0.8680672268907563, "English,Vietnamese": 0.8411764705882353, "Chinese,Vietnamese": 0.819327731092437 }, "3_combine": { "Spanish,English,Chinese": 0.8327731092436975, "Spanish,English,Vietnamese": 0.8109243697478992, "Spanish,Chinese,Vietnamese": 0.7739495798319328, "English,Chinese,Vietnamese": 0.7756302521008404 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7605042016806722 } }, "AC3_2": 0.8709701018171236, "AC3_3": 0.838803670306605, "AC3_4": 0.8174496989712126 }, "prompt_2": { "overall_acc": 0.8821428571428572, "language_acc": { "Spanish": 0.9042016806722689, "English": 0.9184873949579831, "Chinese": 0.8663865546218488, "Vietnamese": 0.8394957983193277 }, "consistency_score_2": 0.8579831932773109, "consistency_score_3": 0.7970588235294117, "consistency_score_4": 0.7571428571428571, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9159663865546218, "Spanish,Chinese": 0.8638655462184874, "Spanish,Vietnamese": 0.8394957983193277, "English,Chinese": 0.8672268907563025, "English,Vietnamese": 0.8378151260504202, "Chinese,Vietnamese": 0.8235294117647058 }, "3_combine": { "Spanish,English,Chinese": 0.8327731092436975, "Spanish,English,Vietnamese": 0.8058823529411765, "Spanish,Chinese,Vietnamese": 0.7747899159663866, "English,Chinese,Vietnamese": 0.7747899159663866 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7571428571428571 } }, "AC3_2": 0.8698953104827705, "AC3_3": 0.837445264556659, "AC3_4": 0.8148770618861761 }, "prompt_3": { "overall_acc": 0.8815126050420169, "language_acc": { "Spanish": 0.9, "English": 0.9159663865546218, "Chinese": 0.8697478991596639, "Vietnamese": 0.8403361344537815 }, "consistency_score_2": 0.8579831932773109, "consistency_score_3": 0.7964285714285714, "consistency_score_4": 0.7571428571428571, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.907563025210084, "Spanish,Chinese": 0.8638655462184874, "Spanish,Vietnamese": 0.8403361344537815, "English,Chinese": 0.8689075630252101, "English,Vietnamese": 0.8369747899159664, "Chinese,Vietnamese": 0.8302521008403362 }, "3_combine": { "Spanish,English,Chinese": 0.8285714285714286, "Spanish,English,Vietnamese": 0.8025210084033614, "Spanish,Chinese,Vietnamese": 0.7773109243697479, "English,Chinese,Vietnamese": 0.7773109243697479 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7571428571428571 } }, "AC3_2": 0.8695887629914576, "AC3_3": 0.8368133931426378, "AC3_4": 0.8146080585583467 }, "prompt_4": { "overall_acc": 0.88109243697479, "language_acc": { "Spanish": 0.8991596638655462, "English": 0.9126050420168067, "Chinese": 0.8714285714285714, "Vietnamese": 0.8411764705882353 }, "consistency_score_2": 0.8586834733893557, "consistency_score_3": 0.7972689075630252, "consistency_score_4": 0.7571428571428571, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9109243697478991, "Spanish,Chinese": 0.873109243697479, "Spanish,Vietnamese": 0.8445378151260504, "English,Chinese": 0.865546218487395, "English,Vietnamese": 0.8336134453781513, "Chinese,Vietnamese": 0.8243697478991596 }, "3_combine": { "Spanish,English,Chinese": 0.8327731092436975, "Spanish,English,Vietnamese": 0.8042016806722689, "Spanish,Chinese,Vietnamese": 0.780672268907563, "English,Chinese,Vietnamese": 0.7714285714285715 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7571428571428571 } }, "AC3_2": 0.8697436372213716, "AC3_3": 0.8370874447665793, "AC3_4": 0.8144286080182891 }, "prompt_5": { "overall_acc": 0.8741596638655463, "language_acc": { "Spanish": 0.8957983193277311, "English": 0.9025210084033614, "Chinese": 0.8689075630252101, "Vietnamese": 0.8294117647058824 }, "consistency_score_2": 0.85, "consistency_score_3": 0.7852941176470588, "consistency_score_4": 0.7411764705882353, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9008403361344538, "Spanish,Chinese": 0.8739495798319328, "Spanish,Vietnamese": 0.8302521008403362, "English,Chinese": 0.8630252100840337, "English,Vietnamese": 0.8168067226890756, "Chinese,Vietnamese": 0.8151260504201681 }, "3_combine": { "Spanish,English,Chinese": 0.8260504201680672, "Spanish,English,Vietnamese": 0.7848739495798319, "Spanish,Chinese,Vietnamese": 0.7714285714285715, "English,Chinese,Vietnamese": 0.7588235294117647 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7411764705882353 } }, "AC3_2": 0.8619105641025625, "AC3_3": 0.8273474676117238, "AC3_4": 0.8021941198925023 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.5038095238095238, "language_acc": { "Filipino": 0.42, "Vietnamese": 0.38666666666666666, "Chinese": 0.5133333333333333, "Spanish": 0.54, "Malay": 0.46, "Indonesian": 0.54, "English": 0.6666666666666666 }, "consistency_score_2": 0.5968253968253967, "consistency_score_3": 0.4401904761904763, "consistency_score_4": 0.3518095238095238, "consistency_score_5": 0.29555555555555557, "consistency_score_6": 0.2580952380952381, "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5, "Filipino,Chinese": 0.5333333333333333, "Filipino,Spanish": 0.5266666666666666, "Filipino,Malay": 0.4866666666666667, "Filipino,Indonesian": 0.54, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.5533333333333333, "Vietnamese,Spanish": 0.5066666666666667, "Vietnamese,Malay": 0.58, "Vietnamese,Indonesian": 0.5466666666666666, "Vietnamese,English": 0.46, "Chinese,Spanish": 0.6133333333333333, "Chinese,Malay": 0.6333333333333333, "Chinese,Indonesian": 0.7066666666666667, "Chinese,English": 0.6333333333333333, "Spanish,Malay": 0.68, "Spanish,Indonesian": 0.7466666666666667, "Spanish,English": 0.74, "Malay,Indonesian": 0.7333333333333333, "Malay,English": 0.62, "Indonesian,English": 0.7 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.36666666666666664, "Filipino,Vietnamese,Spanish": 0.3333333333333333, "Filipino,Vietnamese,Malay": 0.3466666666666667, "Filipino,Vietnamese,Indonesian": 0.34, "Filipino,Vietnamese,English": 0.31333333333333335, "Filipino,Chinese,Spanish": 0.38666666666666666, "Filipino,Chinese,Malay": 0.38, "Filipino,Chinese,Indonesian": 0.4266666666666667, "Filipino,Chinese,English": 0.38666666666666666, "Filipino,Spanish,Malay": 0.3933333333333333, "Filipino,Spanish,Indonesian": 0.43333333333333335, "Filipino,Spanish,English": 0.4266666666666667, "Filipino,Malay,Indonesian": 0.41333333333333333, "Filipino,Malay,English": 0.37333333333333335, "Filipino,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Spanish": 0.4, "Vietnamese,Chinese,Malay": 0.43333333333333335, "Vietnamese,Chinese,Indonesian": 0.44666666666666666, "Vietnamese,Chinese,English": 0.38666666666666666, "Vietnamese,Spanish,Malay": 0.4266666666666667, "Vietnamese,Spanish,Indonesian": 0.44666666666666666, "Vietnamese,Spanish,English": 0.4, "Vietnamese,Malay,Indonesian": 0.47333333333333333, "Vietnamese,Malay,English": 0.4, "Vietnamese,Indonesian,English": 0.3933333333333333, "Chinese,Spanish,Malay": 0.49333333333333335, "Chinese,Spanish,Indonesian": 0.56, "Chinese,Spanish,English": 0.52, "Chinese,Malay,Indonesian": 0.5533333333333333, "Chinese,Malay,English": 0.48, "Chinese,Indonesian,English": 0.5466666666666666, "Spanish,Malay,Indonesian": 0.6066666666666667, "Spanish,Malay,English": 0.54, "Spanish,Indonesian,English": 0.6266666666666667, "Malay,Indonesian,English": 0.56 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Chinese,English": 0.2866666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.3, "Filipino,Vietnamese,Spanish,English": 0.2733333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Malay,English": 0.2733333333333333, "Filipino,Vietnamese,Indonesian,English": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay": 0.32, "Filipino,Chinese,Spanish,Indonesian": 0.35333333333333333, "Filipino,Chinese,Spanish,English": 0.3333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.3466666666666667, "Filipino,Chinese,Malay,English": 0.31333333333333335, "Filipino,Chinese,Indonesian,English": 0.3333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Spanish,Malay,English": 0.3333333333333333, "Filipino,Spanish,Indonesian,English": 0.36666666666666664, "Filipino,Malay,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish,Malay": 0.36, "Vietnamese,Chinese,Spanish,Indonesian": 0.38666666666666666, "Vietnamese,Chinese,Spanish,English": 0.34, "Vietnamese,Chinese,Malay,Indonesian": 0.4, "Vietnamese,Chinese,Malay,English": 0.3466666666666667, "Vietnamese,Chinese,Indonesian,English": 0.3466666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Spanish,Malay,English": 0.35333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.38, "Vietnamese,Malay,Indonesian,English": 0.35333333333333333, "Chinese,Spanish,Malay,Indonesian": 0.47333333333333333, "Chinese,Spanish,Malay,English": 0.42, "Chinese,Spanish,Indonesian,English": 0.4866666666666667, "Chinese,Malay,Indonesian,English": 0.44, "Spanish,Malay,Indonesian,English": 0.5266666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.28, "Filipino,Vietnamese,Chinese,Spanish,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, "Filipino,Chinese,Spanish,Malay,English": 0.28, "Filipino,Chinese,Spanish,Indonesian,English": 0.30666666666666664, "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36, "Vietnamese,Chinese,Spanish,Malay,English": 0.31333333333333335, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.32, "Vietnamese,Spanish,Malay,Indonesian,English": 0.3466666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.41333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, "AC3_2": 0.5463870231761457, "AC3_3": 0.469856258837963, "AC3_4": 0.41430818805317654, "AC3_5": 0.37255493773470155, "AC3_6": 0.34133095233615274, "AC3_7": 0.3189491816966112 }, "prompt_2": { "overall_acc": 0.5038095238095238, "language_acc": { "Filipino": 0.4666666666666667, "Vietnamese": 0.3933333333333333, "Chinese": 0.52, "Spanish": 0.52, "Malay": 0.46, "Indonesian": 0.5333333333333333, "English": 0.6333333333333333 }, "consistency_score_2": 0.5796825396825397, "consistency_score_3": 0.4180952380952379, "consistency_score_4": 0.32800000000000007, "consistency_score_5": 0.2714285714285714, "consistency_score_6": 0.23333333333333334, "consistency_score_7": 0.20666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.47333333333333333, "Filipino,Chinese": 0.5333333333333333, "Filipino,Spanish": 0.5466666666666666, "Filipino,Malay": 0.5, "Filipino,Indonesian": 0.56, "Filipino,English": 0.54, "Vietnamese,Chinese": 0.52, "Vietnamese,Spanish": 0.49333333333333335, "Vietnamese,Malay": 0.5733333333333334, "Vietnamese,Indonesian": 0.5533333333333333, "Vietnamese,English": 0.46, "Chinese,Spanish": 0.6066666666666667, "Chinese,Malay": 0.6, "Chinese,Indonesian": 0.64, "Chinese,English": 0.64, "Spanish,Malay": 0.6333333333333333, "Spanish,Indonesian": 0.7, "Spanish,English": 0.6533333333333333, "Malay,Indonesian": 0.72, "Malay,English": 0.56, "Indonesian,English": 0.6666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.34, "Filipino,Vietnamese,Spanish": 0.32, "Filipino,Vietnamese,Malay": 0.3466666666666667, "Filipino,Vietnamese,Indonesian": 0.36, "Filipino,Vietnamese,English": 0.32, "Filipino,Chinese,Spanish": 0.38, "Filipino,Chinese,Malay": 0.36, "Filipino,Chinese,Indonesian": 0.38666666666666666, "Filipino,Chinese,English": 0.4, "Filipino,Spanish,Malay": 0.4, "Filipino,Spanish,Indonesian": 0.44666666666666666, "Filipino,Spanish,English": 0.43333333333333335, "Filipino,Malay,Indonesian": 0.44, "Filipino,Malay,English": 0.36666666666666664, "Filipino,Indonesian,English": 0.42, "Vietnamese,Chinese,Spanish": 0.38, "Vietnamese,Chinese,Malay": 0.4, "Vietnamese,Chinese,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,English": 0.38, "Vietnamese,Spanish,Malay": 0.4066666666666667, "Vietnamese,Spanish,Indonesian": 0.4266666666666667, "Vietnamese,Spanish,English": 0.36, "Vietnamese,Malay,Indonesian": 0.46, "Vietnamese,Malay,English": 0.36, "Vietnamese,Indonesian,English": 0.3933333333333333, "Chinese,Spanish,Malay": 0.4533333333333333, "Chinese,Spanish,Indonesian": 0.5066666666666667, "Chinese,Spanish,English": 0.47333333333333333, "Chinese,Malay,Indonesian": 0.5066666666666667, "Chinese,Malay,English": 0.44, "Chinese,Indonesian,English": 0.5133333333333333, "Spanish,Malay,Indonesian": 0.5466666666666666, "Spanish,Malay,English": 0.4533333333333333, "Spanish,Indonesian,English": 0.5333333333333333, "Malay,Indonesian,English": 0.5066666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.28, "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Spanish,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian": 0.32, "Filipino,Vietnamese,Malay,English": 0.26, "Filipino,Vietnamese,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay": 0.31333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Chinese,Spanish,English": 0.32, "Filipino,Chinese,Malay,Indonesian": 0.32, "Filipino,Chinese,Malay,English": 0.29333333333333333, "Filipino,Chinese,Indonesian,English": 0.32666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Spanish,Malay,English": 0.32, "Filipino,Spanish,Indonesian,English": 0.36, "Filipino,Malay,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.36, "Vietnamese,Chinese,Spanish,English": 0.30666666666666664, "Vietnamese,Chinese,Malay,Indonesian": 0.36666666666666664, "Vietnamese,Chinese,Malay,English": 0.30666666666666664, "Vietnamese,Chinese,Indonesian,English": 0.34, "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Spanish,Malay,English": 0.30666666666666664, "Vietnamese,Spanish,Indonesian,English": 0.3333333333333333, "Vietnamese,Malay,Indonesian,English": 0.3466666666666667, "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Chinese,Spanish,Malay,English": 0.36666666666666664, "Chinese,Spanish,Indonesian,English": 0.42, "Chinese,Malay,Indonesian,English": 0.4, "Spanish,Malay,Indonesian,English": 0.42 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26, "Filipino,Vietnamese,Chinese,Spanish,English": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Chinese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.24, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.26, "Filipino,Chinese,Spanish,Indonesian,English": 0.28, "Filipino,Chinese,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32, "Vietnamese,Chinese,Spanish,Malay,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3, "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, "Vietnamese,Spanish,Malay,Indonesian,English": 0.3, "Chinese,Spanish,Malay,Indonesian,English": 0.34 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.22, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 } }, "AC3_2": 0.5390894757640105, "AC3_3": 0.4569677291908886, "AC3_4": 0.3973253949602483, "AC3_5": 0.35279045274494536, "AC3_6": 0.3189491816966112, "AC3_7": 0.2931009829792999 }, "prompt_3": { "overall_acc": 0.4942857142857143, "language_acc": { "Filipino": 0.4, "Vietnamese": 0.4266666666666667, "Chinese": 0.47333333333333333, "Spanish": 0.56, "Malay": 0.47333333333333333, "Indonesian": 0.54, "English": 0.5866666666666667 }, "consistency_score_2": 0.5203174603174604, "consistency_score_3": 0.3432380952380952, "consistency_score_4": 0.2521904761904762, "consistency_score_5": 0.19777777777777777, "consistency_score_6": 0.16285714285714284, "consistency_score_7": 0.14, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.46, "Filipino,Chinese": 0.4, "Filipino,Spanish": 0.5266666666666666, "Filipino,Malay": 0.42, "Filipino,Indonesian": 0.43333333333333335, "Filipino,English": 0.41333333333333333, "Vietnamese,Chinese": 0.5133333333333333, "Vietnamese,Spanish": 0.43333333333333335, "Vietnamese,Malay": 0.5533333333333333, "Vietnamese,Indonesian": 0.5133333333333333, "Vietnamese,English": 0.43333333333333335, "Chinese,Spanish": 0.5133333333333333, "Chinese,Malay": 0.5533333333333333, "Chinese,Indonesian": 0.6, "Chinese,English": 0.5466666666666666, "Spanish,Malay": 0.6333333333333333, "Spanish,Indonesian": 0.6266666666666667, "Spanish,English": 0.6066666666666667, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.48, "Indonesian,English": 0.62 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2866666666666667, "Filipino,Vietnamese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Malay": 0.29333333333333333, "Filipino,Vietnamese,Indonesian": 0.28, "Filipino,Vietnamese,English": 0.24, "Filipino,Chinese,Spanish": 0.26666666666666666, "Filipino,Chinese,Malay": 0.2733333333333333, "Filipino,Chinese,Indonesian": 0.2733333333333333, "Filipino,Chinese,English": 0.25333333333333335, "Filipino,Spanish,Malay": 0.3466666666666667, "Filipino,Spanish,Indonesian": 0.35333333333333333, "Filipino,Spanish,English": 0.3333333333333333, "Filipino,Malay,Indonesian": 0.32666666666666666, "Filipino,Malay,English": 0.25333333333333335, "Filipino,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish": 0.29333333333333333, "Vietnamese,Chinese,Malay": 0.36, "Vietnamese,Chinese,Indonesian": 0.37333333333333335, "Vietnamese,Chinese,English": 0.32, "Vietnamese,Spanish,Malay": 0.36, "Vietnamese,Spanish,Indonesian": 0.36, "Vietnamese,Spanish,English": 0.30666666666666664, "Vietnamese,Malay,Indonesian": 0.42, "Vietnamese,Malay,English": 0.31333333333333335, "Vietnamese,Indonesian,English": 0.34, "Chinese,Spanish,Malay": 0.3933333333333333, "Chinese,Spanish,Indonesian": 0.42, "Chinese,Spanish,English": 0.38, "Chinese,Malay,Indonesian": 0.44, "Chinese,Malay,English": 0.34, "Chinese,Indonesian,English": 0.44, "Spanish,Malay,Indonesian": 0.49333333333333335, "Spanish,Malay,English": 0.4066666666666667, "Spanish,Indonesian,English": 0.47333333333333333, "Malay,Indonesian,English": 0.4266666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Chinese,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.24, "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Chinese,Malay,English": 0.19333333333333333, "Filipino,Chinese,Indonesian,English": 0.20666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.29333333333333333, "Filipino,Spanish,Malay,English": 0.22, "Filipino,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.26, "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Spanish,English": 0.24, "Vietnamese,Chinese,Malay,Indonesian": 0.32, "Vietnamese,Chinese,Malay,English": 0.25333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.2866666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Spanish,Malay,English": 0.26, "Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, "Vietnamese,Malay,Indonesian,English": 0.28, "Chinese,Spanish,Malay,Indonesian": 0.3466666666666667, "Chinese,Spanish,Malay,English": 0.29333333333333333, "Chinese,Spanish,Indonesian,English": 0.3466666666666667, "Chinese,Malay,Indonesian,English": 0.32, "Spanish,Malay,Indonesian,English": 0.37333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.18, "Filipino,Chinese,Malay,Indonesian,English": 0.18, "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.28 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14 } }, "AC3_2": 0.5069676380691103, "AC3_3": 0.4051411676308881, "AC3_4": 0.3339802427285613, "AC3_5": 0.28251376142706797, "AC3_6": 0.2449937887825941, "AC3_7": 0.2181981981637976 }, "prompt_4": { "overall_acc": 0.499047619047619, "language_acc": { "Filipino": 0.43333333333333335, "Vietnamese": 0.38666666666666666, "Chinese": 0.5133333333333333, "Spanish": 0.5266666666666666, "Malay": 0.4866666666666667, "Indonesian": 0.5333333333333333, "English": 0.6133333333333333 }, "consistency_score_2": 0.5647619047619048, "consistency_score_3": 0.3992380952380954, "consistency_score_4": 0.30876190476190474, "consistency_score_5": 0.25238095238095243, "consistency_score_6": 0.21428571428571427, "consistency_score_7": 0.18666666666666668, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4533333333333333, "Filipino,Chinese": 0.4866666666666667, "Filipino,Spanish": 0.4866666666666667, "Filipino,Malay": 0.4666666666666667, "Filipino,Indonesian": 0.5066666666666667, "Filipino,English": 0.46, "Vietnamese,Chinese": 0.5333333333333333, "Vietnamese,Spanish": 0.4866666666666667, "Vietnamese,Malay": 0.5733333333333334, "Vietnamese,Indonesian": 0.5466666666666666, "Vietnamese,English": 0.46, "Chinese,Spanish": 0.5733333333333334, "Chinese,Malay": 0.58, "Chinese,Indonesian": 0.6466666666666666, "Chinese,English": 0.58, "Spanish,Malay": 0.6466666666666666, "Spanish,Indonesian": 0.74, "Spanish,English": 0.6666666666666666, "Malay,Indonesian": 0.7333333333333333, "Malay,English": 0.56, "Indonesian,English": 0.6733333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.32666666666666666, "Filipino,Vietnamese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Malay": 0.30666666666666664, "Filipino,Vietnamese,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,English": 0.28, "Filipino,Chinese,Spanish": 0.31333333333333335, "Filipino,Chinese,Malay": 0.34, "Filipino,Chinese,Indonesian": 0.36, "Filipino,Chinese,English": 0.32, "Filipino,Spanish,Malay": 0.36, "Filipino,Spanish,Indonesian": 0.4066666666666667, "Filipino,Spanish,English": 0.36, "Filipino,Malay,Indonesian": 0.4, "Filipino,Malay,English": 0.32, "Filipino,Indonesian,English": 0.36, "Vietnamese,Chinese,Spanish": 0.36, "Vietnamese,Chinese,Malay": 0.4, "Vietnamese,Chinese,Indonesian": 0.4066666666666667, "Vietnamese,Chinese,English": 0.35333333333333333, "Vietnamese,Spanish,Malay": 0.3933333333333333, "Vietnamese,Spanish,Indonesian": 0.43333333333333335, "Vietnamese,Spanish,English": 0.36666666666666664, "Vietnamese,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Malay,English": 0.38, "Vietnamese,Indonesian,English": 0.4, "Chinese,Spanish,Malay": 0.43333333333333335, "Chinese,Spanish,Indonesian": 0.5133333333333333, "Chinese,Spanish,English": 0.46, "Chinese,Malay,Indonesian": 0.5, "Chinese,Malay,English": 0.4066666666666667, "Chinese,Indonesian,English": 0.49333333333333335, "Spanish,Malay,Indonesian": 0.58, "Spanish,Malay,English": 0.48, "Spanish,Indonesian,English": 0.5666666666666667, "Malay,Indonesian,English": 0.5266666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.24, "Filipino,Vietnamese,Chinese,Malay": 0.26, "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,English": 0.24, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Spanish,English": 0.22666666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay": 0.26666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Chinese,Spanish,English": 0.26, "Filipino,Chinese,Malay,Indonesian": 0.3, "Filipino,Chinese,Malay,English": 0.25333333333333335, "Filipino,Chinese,Indonesian,English": 0.2866666666666667, "Filipino,Spanish,Malay,Indonesian": 0.34, "Filipino,Spanish,Malay,English": 0.28, "Filipino,Spanish,Indonesian,English": 0.31333333333333335, "Filipino,Malay,Indonesian,English": 0.31333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Indonesian": 0.34, "Vietnamese,Chinese,Spanish,English": 0.29333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,Malay,English": 0.3, "Vietnamese,Chinese,Indonesian,English": 0.32666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Spanish,Malay,English": 0.32666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.35333333333333333, "Vietnamese,Malay,Indonesian,English": 0.36, "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Chinese,Spanish,Malay,English": 0.36666666666666664, "Chinese,Spanish,Indonesian,English": 0.4266666666666667, "Chinese,Malay,Indonesian,English": 0.4, "Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,English": 0.2, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.2, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26, "Filipino,Chinese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Spanish,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3, "Vietnamese,Chinese,Spanish,Malay,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, "Vietnamese,Spanish,Malay,Indonesian,English": 0.32, "Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.2, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 } }, "AC3_2": 0.5298750905414855, "AC3_3": 0.443597883548501, "AC3_4": 0.3814931338722228, "AC3_5": 0.3352284386058798, "AC3_6": 0.29982834250994206, "AC3_7": 0.27170370366408025 }, "prompt_5": { "overall_acc": 0.49714285714285716, "language_acc": { "Filipino": 0.41333333333333333, "Vietnamese": 0.4, "Chinese": 0.49333333333333335, "Spanish": 0.5133333333333333, "Malay": 0.5, "Indonesian": 0.54, "English": 0.62 }, "consistency_score_2": 0.5552380952380952, "consistency_score_3": 0.3872380952380953, "consistency_score_4": 0.29999999999999993, "consistency_score_5": 0.2485714285714286, "consistency_score_6": 0.2161904761904762, "consistency_score_7": 0.19333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4866666666666667, "Filipino,Chinese": 0.49333333333333335, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.5133333333333333, "Filipino,Indonesian": 0.4533333333333333, "Filipino,English": 0.47333333333333333, "Vietnamese,Chinese": 0.5, "Vietnamese,Spanish": 0.49333333333333335, "Vietnamese,Malay": 0.6, "Vietnamese,Indonesian": 0.58, "Vietnamese,English": 0.44, "Chinese,Spanish": 0.5333333333333333, "Chinese,Malay": 0.58, "Chinese,Indonesian": 0.6466666666666666, "Chinese,English": 0.5466666666666666, "Spanish,Malay": 0.66, "Spanish,Indonesian": 0.6866666666666666, "Spanish,English": 0.62, "Malay,Indonesian": 0.7333333333333333, "Malay,English": 0.5333333333333333, "Indonesian,English": 0.6066666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.32, "Filipino,Vietnamese,Spanish": 0.3, "Filipino,Vietnamese,Malay": 0.35333333333333333, "Filipino,Vietnamese,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,English": 0.28, "Filipino,Chinese,Spanish": 0.32, "Filipino,Chinese,Malay": 0.36, "Filipino,Chinese,Indonesian": 0.3466666666666667, "Filipino,Chinese,English": 0.32666666666666666, "Filipino,Spanish,Malay": 0.38, "Filipino,Spanish,Indonesian": 0.37333333333333335, "Filipino,Spanish,English": 0.32, "Filipino,Malay,Indonesian": 0.4, "Filipino,Malay,English": 0.32666666666666666, "Filipino,Indonesian,English": 0.31333333333333335, "Vietnamese,Chinese,Spanish": 0.3466666666666667, "Vietnamese,Chinese,Malay": 0.41333333333333333, "Vietnamese,Chinese,Indonesian": 0.42, "Vietnamese,Chinese,English": 0.34, "Vietnamese,Spanish,Malay": 0.42, "Vietnamese,Spanish,Indonesian": 0.4266666666666667, "Vietnamese,Spanish,English": 0.3333333333333333, "Vietnamese,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Malay,English": 0.3466666666666667, "Vietnamese,Indonesian,English": 0.36666666666666664, "Chinese,Spanish,Malay": 0.43333333333333335, "Chinese,Spanish,Indonesian": 0.47333333333333333, "Chinese,Spanish,English": 0.4066666666666667, "Chinese,Malay,Indonesian": 0.5133333333333333, "Chinese,Malay,English": 0.38, "Chinese,Indonesian,English": 0.44, "Spanish,Malay,Indonesian": 0.5666666666666667, "Spanish,Malay,English": 0.43333333333333335, "Spanish,Indonesian,English": 0.49333333333333335, "Malay,Indonesian,English": 0.4666666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.28, "Filipino,Vietnamese,Chinese,Indonesian": 0.28, "Filipino,Vietnamese,Chinese,English": 0.24, "Filipino,Vietnamese,Spanish,Malay": 0.28, "Filipino,Vietnamese,Spanish,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay": 0.2866666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Chinese,Spanish,English": 0.25333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, "Filipino,Chinese,Malay,English": 0.25333333333333335, "Filipino,Chinese,Indonesian,English": 0.25333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.3466666666666667, "Filipino,Spanish,Malay,English": 0.26, "Filipino,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.32666666666666666, "Vietnamese,Chinese,Spanish,English": 0.2733333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Chinese,Malay,English": 0.3, "Vietnamese,Chinese,Indonesian,English": 0.31333333333333335, "Vietnamese,Spanish,Malay,Indonesian": 0.38, "Vietnamese,Spanish,Malay,English": 0.2866666666666667, "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, "Vietnamese,Malay,Indonesian,English": 0.31333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Chinese,Spanish,Malay,English": 0.32, "Chinese,Spanish,Indonesian,English": 0.36666666666666664, "Chinese,Malay,Indonesian,English": 0.35333333333333333, "Spanish,Malay,Indonesian,English": 0.4066666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.2, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.22, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.21333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.28, "Filipino,Chinese,Spanish,Malay,English": 0.22, "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, "Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.2866666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.28, "Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.25333333333333335 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 } }, "AC3_2": 0.5245869424194477, "AC3_3": 0.43536137344697234, "AC3_4": 0.37419354834015484, "AC3_5": 0.331428571384127, "AC3_6": 0.30133892805235857, "AC3_7": 0.27839999995968 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.452922077922078, "language_acc": { "Indonesian": 0.5, "English": 0.5397727272727273, "Filipino": 0.3522727272727273, "Spanish": 0.4943181818181818, "Chinese": 0.48863636363636365, "Malay": 0.4147727272727273, "Vietnamese": 0.3806818181818182 }, "consistency_score_2": 0.5662878787878788, "consistency_score_3": 0.39577922077922073, "consistency_score_4": 0.3006493506493506, "consistency_score_5": 0.23917748917748916, "consistency_score_6": 0.19642857142857142, "consistency_score_7": 0.16477272727272727, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7215909090909091, "Indonesian,Filipino": 0.4943181818181818, "Indonesian,Spanish": 0.7443181818181818, "Indonesian,Chinese": 0.6420454545454546, "Indonesian,Malay": 0.7045454545454546, "Indonesian,Vietnamese": 0.5056818181818182, "English,Filipino": 0.45454545454545453, "English,Spanish": 0.6875, "English,Chinese": 0.6306818181818182, "English,Malay": 0.6079545454545454, "English,Vietnamese": 0.48295454545454547, "Filipino,Spanish": 0.45454545454545453, "Filipino,Chinese": 0.4318181818181818, "Filipino,Malay": 0.5, "Filipino,Vietnamese": 0.5170454545454546, "Spanish,Chinese": 0.6136363636363636, "Spanish,Malay": 0.6079545454545454, "Spanish,Vietnamese": 0.5113636363636364, "Chinese,Malay": 0.5852272727272727, "Chinese,Vietnamese": 0.48295454545454547, "Malay,Vietnamese": 0.5113636363636364 }, "3_combine": { "Indonesian,English,Filipino": 0.38636363636363635, "Indonesian,English,Spanish": 0.5909090909090909, "Indonesian,English,Chinese": 0.5340909090909091, "Indonesian,English,Malay": 0.5454545454545454, "Indonesian,English,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish": 0.38636363636363635, "Indonesian,Filipino,Chinese": 0.3522727272727273, "Indonesian,Filipino,Malay": 0.4034090909090909, "Indonesian,Filipino,Vietnamese": 0.3181818181818182, "Indonesian,Spanish,Chinese": 0.5227272727272727, "Indonesian,Spanish,Malay": 0.5454545454545454, "Indonesian,Spanish,Vietnamese": 0.4090909090909091, "Indonesian,Chinese,Malay": 0.48863636363636365, "Indonesian,Chinese,Vietnamese": 0.3522727272727273, "Indonesian,Malay,Vietnamese": 0.3977272727272727, "English,Filipino,Spanish": 0.3465909090909091, "English,Filipino,Chinese": 0.3352272727272727, "English,Filipino,Malay": 0.3465909090909091, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.5113636363636364, "English,Spanish,Malay": 0.48295454545454547, "English,Spanish,Vietnamese": 0.3806818181818182, "English,Chinese,Malay": 0.45454545454545453, "English,Chinese,Vietnamese": 0.35795454545454547, "English,Malay,Vietnamese": 0.3522727272727273, "Filipino,Spanish,Chinese": 0.3181818181818182, "Filipino,Spanish,Malay": 0.3409090909090909, "Filipino,Spanish,Vietnamese": 0.3068181818181818, "Filipino,Chinese,Malay": 0.3181818181818182, "Filipino,Chinese,Vietnamese": 0.2840909090909091, "Filipino,Malay,Vietnamese": 0.3125, "Spanish,Chinese,Malay": 0.4375, "Spanish,Chinese,Vietnamese": 0.3522727272727273, "Spanish,Malay,Vietnamese": 0.36363636363636365, "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.3181818181818182, "Indonesian,English,Filipino,Chinese": 0.3125, "Indonesian,English,Filipino,Malay": 0.32386363636363635, "Indonesian,English,Filipino,Vietnamese": 0.25, "Indonesian,English,Spanish,Chinese": 0.4659090909090909, "Indonesian,English,Spanish,Malay": 0.45454545454545453, "Indonesian,English,Spanish,Vietnamese": 0.3352272727272727, "Indonesian,English,Chinese,Malay": 0.42045454545454547, "Indonesian,English,Chinese,Vietnamese": 0.30113636363636365, "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.29545454545454547, "Indonesian,Filipino,Spanish,Malay": 0.3181818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Chinese,Malay": 0.30113636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Spanish,Chinese,Malay": 0.3977272727272727, "Indonesian,Spanish,Chinese,Vietnamese": 0.30113636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.32386363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish,Chinese": 0.29545454545454547, "English,Filipino,Spanish,Malay": 0.2727272727272727, "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, "English,Filipino,Chinese,Malay": 0.2784090909090909, "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, "English,Filipino,Malay,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese,Malay": 0.38636363636363635, "English,Spanish,Chinese,Vietnamese": 0.30113636363636365, "English,Spanish,Malay,Vietnamese": 0.29545454545454547, "English,Chinese,Malay,Vietnamese": 0.2727272727272727, "Filipino,Spanish,Chinese,Malay": 0.2556818181818182, "Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay,Vietnamese": 0.23295454545454544, "Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2784090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.26704545454545453, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "Indonesian,English,Filipino,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.2159090909090909, "Indonesian,English,Spanish,Chinese,Malay": 0.36363636363636365, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.25, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese,Malay": 0.23863636363636365, "English,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, "English,Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1875 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 } }, "AC3_2": 0.5033001906014862, "AC3_3": 0.42242694179185863, "AC3_4": 0.36140098586832264, "AC3_5": 0.3130438755475109, "AC3_6": 0.2740178571006584, "AC3_7": 0.24163779711773178 }, "prompt_2": { "overall_acc": 0.43262987012987014, "language_acc": { "Indonesian": 0.48863636363636365, "English": 0.4772727272727273, "Filipino": 0.3465909090909091, "Spanish": 0.4943181818181818, "Chinese": 0.48863636363636365, "Malay": 0.4090909090909091, "Vietnamese": 0.32386363636363635 }, "consistency_score_2": 0.5546536796536797, "consistency_score_3": 0.37727272727272726, "consistency_score_4": 0.274025974025974, "consistency_score_5": 0.20427489177489178, "consistency_score_6": 0.15503246753246755, "consistency_score_7": 0.11931818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6931818181818182, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.7159090909090909, "Indonesian,Chinese": 0.6363636363636364, "Indonesian,Malay": 0.6988636363636364, "Indonesian,Vietnamese": 0.5056818181818182, "English,Filipino": 0.4602272727272727, "English,Spanish": 0.6420454545454546, "English,Chinese": 0.5965909090909091, "English,Malay": 0.5965909090909091, "English,Vietnamese": 0.44886363636363635, "Filipino,Spanish": 0.4772727272727273, "Filipino,Chinese": 0.45454545454545453, "Filipino,Malay": 0.5056818181818182, "Filipino,Vietnamese": 0.5284090909090909, "Spanish,Chinese": 0.5681818181818182, "Spanish,Malay": 0.6477272727272727, "Spanish,Vietnamese": 0.48295454545454547, "Chinese,Malay": 0.5795454545454546, "Chinese,Vietnamese": 0.4318181818181818, "Malay,Vietnamese": 0.5056818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.36363636363636365, "Indonesian,English,Spanish": 0.5568181818181818, "Indonesian,English,Chinese": 0.5, "Indonesian,English,Malay": 0.5170454545454546, "Indonesian,English,Vietnamese": 0.3522727272727273, "Indonesian,Filipino,Spanish": 0.375, "Indonesian,Filipino,Chinese": 0.32386363636363635, "Indonesian,Filipino,Malay": 0.39204545454545453, "Indonesian,Filipino,Vietnamese": 0.3125, "Indonesian,Spanish,Chinese": 0.48863636363636365, "Indonesian,Spanish,Malay": 0.5454545454545454, "Indonesian,Spanish,Vietnamese": 0.39204545454545453, "Indonesian,Chinese,Malay": 0.48295454545454547, "Indonesian,Chinese,Vietnamese": 0.32386363636363635, "Indonesian,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish": 0.3409090909090909, "English,Filipino,Chinese": 0.32386363636363635, "English,Filipino,Malay": 0.3465909090909091, "English,Filipino,Vietnamese": 0.2784090909090909, "English,Spanish,Chinese": 0.45454545454545453, "English,Spanish,Malay": 0.4659090909090909, "English,Spanish,Vietnamese": 0.3352272727272727, "English,Chinese,Malay": 0.42045454545454547, "English,Chinese,Vietnamese": 0.3068181818181818, "English,Malay,Vietnamese": 0.32954545454545453, "Filipino,Spanish,Chinese": 0.30113636363636365, "Filipino,Spanish,Malay": 0.3693181818181818, "Filipino,Spanish,Vietnamese": 0.3181818181818182, "Filipino,Chinese,Malay": 0.3125, "Filipino,Chinese,Vietnamese": 0.2840909090909091, "Filipino,Malay,Vietnamese": 0.3181818181818182, "Spanish,Chinese,Malay": 0.42045454545454547, "Spanish,Chinese,Vietnamese": 0.30113636363636365, "Spanish,Malay,Vietnamese": 0.36363636363636365, "Chinese,Malay,Vietnamese": 0.30113636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.29545454545454547, "Indonesian,English,Filipino,Chinese": 0.2840909090909091, "Indonesian,English,Filipino,Malay": 0.30113636363636365, "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Chinese": 0.42045454545454547, "Indonesian,English,Spanish,Malay": 0.4318181818181818, "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay": 0.38636363636363635, "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,English,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Malay": 0.3125, "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Chinese,Malay": 0.2727272727272727, "Indonesian,Filipino,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Chinese,Malay": 0.38636363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Spanish,Chinese": 0.25, "English,Filipino,Spanish,Malay": 0.25, "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "English,Filipino,Chinese,Malay": 0.26136363636363635, "English,Filipino,Chinese,Vietnamese": 0.2159090909090909, "English,Filipino,Malay,Vietnamese": 0.21022727272727273, "English,Spanish,Chinese,Malay": 0.3522727272727273, "English,Spanish,Chinese,Vietnamese": 0.25, "English,Spanish,Malay,Vietnamese": 0.25, "English,Chinese,Malay,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Chinese,Malay": 0.23863636363636365, "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.23295454545454544, "Indonesian,English,Filipino,Spanish,Malay": 0.23863636363636365, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.23863636363636365, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1875, "Indonesian,English,Spanish,Chinese,Malay": 0.3352272727272727, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "English,Spanish,Chinese,Malay,Vietnamese": 0.1875, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 } }, "AC3_2": 0.48610097762477517, "AC3_3": 0.40305945803073795, "AC3_4": 0.3355291618083518, "AC3_5": 0.2775153372105073, "AC3_6": 0.22826603641089505, "AC3_7": 0.187048796757555 }, "prompt_3": { "overall_acc": 0.43749999999999994, "language_acc": { "Indonesian": 0.4659090909090909, "English": 0.4943181818181818, "Filipino": 0.36363636363636365, "Spanish": 0.4715909090909091, "Chinese": 0.4943181818181818, "Malay": 0.4318181818181818, "Vietnamese": 0.3409090909090909 }, "consistency_score_2": 0.5451839826839826, "consistency_score_3": 0.3657467532467532, "consistency_score_4": 0.26233766233766226, "consistency_score_5": 0.19291125541125545, "consistency_score_6": 0.14204545454545453, "consistency_score_7": 0.10227272727272728, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6761363636363636, "Indonesian,Filipino": 0.4772727272727273, "Indonesian,Spanish": 0.7045454545454546, "Indonesian,Chinese": 0.6079545454545454, "Indonesian,Malay": 0.6875, "Indonesian,Vietnamese": 0.5113636363636364, "English,Filipino": 0.4318181818181818, "English,Spanish": 0.6306818181818182, "English,Chinese": 0.5965909090909091, "English,Malay": 0.5511363636363636, "English,Vietnamese": 0.4147727272727273, "Filipino,Spanish": 0.45454545454545453, "Filipino,Chinese": 0.42613636363636365, "Filipino,Malay": 0.5284090909090909, "Filipino,Vietnamese": 0.5227272727272727, "Spanish,Chinese": 0.5852272727272727, "Spanish,Malay": 0.6193181818181818, "Spanish,Vietnamese": 0.5, "Chinese,Malay": 0.5625, "Chinese,Vietnamese": 0.4147727272727273, "Malay,Vietnamese": 0.5454545454545454 }, "3_combine": { "Indonesian,English,Filipino": 0.3465909090909091, "Indonesian,English,Spanish": 0.5227272727272727, "Indonesian,English,Chinese": 0.4715909090909091, "Indonesian,English,Malay": 0.48863636363636365, "Indonesian,English,Vietnamese": 0.3465909090909091, "Indonesian,Filipino,Spanish": 0.3693181818181818, "Indonesian,Filipino,Chinese": 0.32954545454545453, "Indonesian,Filipino,Malay": 0.4034090909090909, "Indonesian,Filipino,Vietnamese": 0.3125, "Indonesian,Spanish,Chinese": 0.4659090909090909, "Indonesian,Spanish,Malay": 0.5227272727272727, "Indonesian,Spanish,Vietnamese": 0.4034090909090909, "Indonesian,Chinese,Malay": 0.4602272727272727, "Indonesian,Chinese,Vietnamese": 0.3068181818181818, "Indonesian,Malay,Vietnamese": 0.4034090909090909, "English,Filipino,Spanish": 0.3125, "English,Filipino,Chinese": 0.29545454545454547, "English,Filipino,Malay": 0.3352272727272727, "English,Filipino,Vietnamese": 0.24431818181818182, "English,Spanish,Chinese": 0.4375, "English,Spanish,Malay": 0.42045454545454547, "English,Spanish,Vietnamese": 0.32386363636363635, "English,Chinese,Malay": 0.39204545454545453, "English,Chinese,Vietnamese": 0.2727272727272727, "English,Malay,Vietnamese": 0.3125, "Filipino,Spanish,Chinese": 0.30113636363636365, "Filipino,Spanish,Malay": 0.36363636363636365, "Filipino,Spanish,Vietnamese": 0.29545454545454547, "Filipino,Chinese,Malay": 0.3181818181818182, "Filipino,Chinese,Vietnamese": 0.2784090909090909, "Filipino,Malay,Vietnamese": 0.3465909090909091, "Spanish,Chinese,Malay": 0.4090909090909091, "Spanish,Chinese,Vietnamese": 0.29545454545454547, "Spanish,Malay,Vietnamese": 0.39204545454545453, "Chinese,Malay,Vietnamese": 0.30113636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.26704545454545453, "Indonesian,English,Filipino,Chinese": 0.26136363636363635, "Indonesian,English,Filipino,Malay": 0.29545454545454547, "Indonesian,English,Filipino,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Chinese": 0.38636363636363635, "Indonesian,English,Spanish,Malay": 0.38636363636363635, "Indonesian,English,Spanish,Vietnamese": 0.2840909090909091, "Indonesian,English,Chinese,Malay": 0.36363636363636365, "Indonesian,English,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,English,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Malay": 0.3125, "Indonesian,Filipino,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Chinese,Malay": 0.2840909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Spanish,Chinese,Malay": 0.36363636363636365, "Indonesian,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.3352272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese": 0.22727272727272727, "English,Filipino,Spanish,Malay": 0.23863636363636365, "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "English,Filipino,Chinese,Malay": 0.25, "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, "English,Filipino,Malay,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese,Malay": 0.3125, "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, "English,Spanish,Malay,Vietnamese": 0.24431818181818182, "English,Chinese,Malay,Vietnamese": 0.2215909090909091, "Filipino,Spanish,Chinese,Malay": 0.25, "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Filipino,Spanish,Malay,Vietnamese": 0.25, "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.21022727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.22727272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.29545454545454547, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Malay,Vietnamese": 0.22727272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.1875, "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 } }, "AC3_2": 0.4854419052369439, "AC3_3": 0.39841855290108613, "AC3_4": 0.3279981442352013, "AC3_5": 0.2677575106871403, "AC3_6": 0.21446078427672047, "AC3_7": 0.16578947365349586 }, "prompt_4": { "overall_acc": 0.439935064935065, "language_acc": { "Indonesian": 0.4715909090909091, "English": 0.5, "Filipino": 0.3522727272727273, "Spanish": 0.48295454545454547, "Chinese": 0.4943181818181818, "Malay": 0.3977272727272727, "Vietnamese": 0.3806818181818182 }, "consistency_score_2": 0.5497835497835497, "consistency_score_3": 0.3689935064935065, "consistency_score_4": 0.2631493506493507, "consistency_score_5": 0.19291125541125545, "consistency_score_6": 0.14366883116883117, "consistency_score_7": 0.10795454545454546, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6534090909090909, "Indonesian,Filipino": 0.5056818181818182, "Indonesian,Spanish": 0.7045454545454546, "Indonesian,Chinese": 0.6420454545454546, "Indonesian,Malay": 0.7102272727272727, "Indonesian,Vietnamese": 0.5568181818181818, "English,Filipino": 0.38636363636363635, "English,Spanish": 0.6079545454545454, "English,Chinese": 0.5738636363636364, "English,Malay": 0.5568181818181818, "English,Vietnamese": 0.4147727272727273, "Filipino,Spanish": 0.4659090909090909, "Filipino,Chinese": 0.4431818181818182, "Filipino,Malay": 0.5, "Filipino,Vietnamese": 0.5, "Spanish,Chinese": 0.6022727272727273, "Spanish,Malay": 0.6079545454545454, "Spanish,Vietnamese": 0.5170454545454546, "Chinese,Malay": 0.5625, "Chinese,Vietnamese": 0.4943181818181818, "Malay,Vietnamese": 0.5397727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.3352272727272727, "Indonesian,English,Spanish": 0.5056818181818182, "Indonesian,English,Chinese": 0.48295454545454547, "Indonesian,English,Malay": 0.4943181818181818, "Indonesian,English,Vietnamese": 0.3465909090909091, "Indonesian,Filipino,Spanish": 0.38636363636363635, "Indonesian,Filipino,Chinese": 0.3409090909090909, "Indonesian,Filipino,Malay": 0.39204545454545453, "Indonesian,Filipino,Vietnamese": 0.32954545454545453, "Indonesian,Spanish,Chinese": 0.5, "Indonesian,Spanish,Malay": 0.5227272727272727, "Indonesian,Spanish,Vietnamese": 0.4090909090909091, "Indonesian,Chinese,Malay": 0.4772727272727273, "Indonesian,Chinese,Vietnamese": 0.375, "Indonesian,Malay,Vietnamese": 0.42613636363636365, "English,Filipino,Spanish": 0.2840909090909091, "English,Filipino,Chinese": 0.2897727272727273, "English,Filipino,Malay": 0.2784090909090909, "English,Filipino,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese": 0.4431818181818182, "English,Spanish,Malay": 0.4147727272727273, "English,Spanish,Vietnamese": 0.3181818181818182, "English,Chinese,Malay": 0.4034090909090909, "English,Chinese,Vietnamese": 0.3068181818181818, "English,Malay,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Chinese": 0.3068181818181818, "Filipino,Spanish,Malay": 0.3352272727272727, "Filipino,Spanish,Vietnamese": 0.30113636363636365, "Filipino,Chinese,Malay": 0.2897727272727273, "Filipino,Chinese,Vietnamese": 0.2840909090909091, "Filipino,Malay,Vietnamese": 0.3181818181818182, "Spanish,Chinese,Malay": 0.42613636363636365, "Spanish,Chinese,Vietnamese": 0.3465909090909091, "Spanish,Malay,Vietnamese": 0.375, "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2556818181818182, "Indonesian,English,Filipino,Chinese": 0.26136363636363635, "Indonesian,English,Filipino,Malay": 0.25, "Indonesian,English,Filipino,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Chinese": 0.3977272727272727, "Indonesian,English,Spanish,Malay": 0.3977272727272727, "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay": 0.375, "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,English,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Malay": 0.30113636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Chinese,Malay": 0.26136363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Spanish,Chinese,Malay": 0.38636363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish,Chinese": 0.2215909090909091, "English,Filipino,Spanish,Malay": 0.20454545454545456, "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, "English,Filipino,Chinese,Malay": 0.2159090909090909, "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, "English,Filipino,Malay,Vietnamese": 0.16477272727272727, "English,Spanish,Chinese,Malay": 0.3352272727272727, "English,Spanish,Chinese,Vietnamese": 0.25, "English,Spanish,Malay,Vietnamese": 0.23863636363636365, "English,Chinese,Malay,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese,Malay": 0.32386363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.125, "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 } }, "AC3_2": 0.488763286964313, "AC3_3": 0.40135356310406817, "AC3_4": 0.3293164350765693, "AC3_5": 0.2682118009825873, "AC3_6": 0.216602243338425, "AC3_7": 0.17336700333536068 }, "prompt_5": { "overall_acc": 0.44237012987012986, "language_acc": { "Indonesian": 0.4772727272727273, "English": 0.5227272727272727, "Filipino": 0.32954545454545453, "Spanish": 0.5056818181818182, "Chinese": 0.48295454545454547, "Malay": 0.39204545454545453, "Vietnamese": 0.38636363636363635 }, "consistency_score_2": 0.5549242424242424, "consistency_score_3": 0.3759740259740261, "consistency_score_4": 0.2725649350649351, "consistency_score_5": 0.2031926406926407, "consistency_score_6": 0.15259740259740262, "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6477272727272727, "Indonesian,Filipino": 0.4602272727272727, "Indonesian,Spanish": 0.7045454545454546, "Indonesian,Chinese": 0.6477272727272727, "Indonesian,Malay": 0.6988636363636364, "Indonesian,Vietnamese": 0.5397727272727273, "English,Filipino": 0.4147727272727273, "English,Spanish": 0.6818181818181818, "English,Chinese": 0.5909090909090909, "English,Malay": 0.5909090909090909, "English,Vietnamese": 0.4772727272727273, "Filipino,Spanish": 0.42613636363636365, "Filipino,Chinese": 0.4318181818181818, "Filipino,Malay": 0.4715909090909091, "Filipino,Vietnamese": 0.5170454545454546, "Spanish,Chinese": 0.5852272727272727, "Spanish,Malay": 0.6306818181818182, "Spanish,Vietnamese": 0.5568181818181818, "Chinese,Malay": 0.5568181818181818, "Chinese,Vietnamese": 0.4943181818181818, "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.32386363636363635, "Indonesian,English,Spanish": 0.5340909090909091, "Indonesian,English,Chinese": 0.48863636363636365, "Indonesian,English,Malay": 0.4943181818181818, "Indonesian,English,Vietnamese": 0.375, "Indonesian,Filipino,Spanish": 0.3352272727272727, "Indonesian,Filipino,Chinese": 0.3125, "Indonesian,Filipino,Malay": 0.375, "Indonesian,Filipino,Vietnamese": 0.3181818181818182, "Indonesian,Spanish,Chinese": 0.5, "Indonesian,Spanish,Malay": 0.5397727272727273, "Indonesian,Spanish,Vietnamese": 0.42045454545454547, "Indonesian,Chinese,Malay": 0.48295454545454547, "Indonesian,Chinese,Vietnamese": 0.3693181818181818, "Indonesian,Malay,Vietnamese": 0.4090909090909091, "English,Filipino,Spanish": 0.3125, "English,Filipino,Chinese": 0.2840909090909091, "English,Filipino,Malay": 0.30113636363636365, "English,Filipino,Vietnamese": 0.2556818181818182, "English,Spanish,Chinese": 0.4659090909090909, "English,Spanish,Malay": 0.48863636363636365, "English,Spanish,Vietnamese": 0.3977272727272727, "English,Chinese,Malay": 0.4034090909090909, "English,Chinese,Vietnamese": 0.3409090909090909, "English,Malay,Vietnamese": 0.3409090909090909, "Filipino,Spanish,Chinese": 0.2727272727272727, "Filipino,Spanish,Malay": 0.3181818181818182, "Filipino,Spanish,Vietnamese": 0.30113636363636365, "Filipino,Chinese,Malay": 0.2784090909090909, "Filipino,Chinese,Vietnamese": 0.3068181818181818, "Filipino,Malay,Vietnamese": 0.3068181818181818, "Spanish,Chinese,Malay": 0.42613636363636365, "Spanish,Chinese,Vietnamese": 0.35795454545454547, "Spanish,Malay,Vietnamese": 0.39204545454545453, "Chinese,Malay,Vietnamese": 0.32954545454545453 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2556818181818182, "Indonesian,English,Filipino,Chinese": 0.2556818181818182, "Indonesian,English,Filipino,Malay": 0.2556818181818182, "Indonesian,English,Filipino,Vietnamese": 0.2215909090909091, "Indonesian,English,Spanish,Chinese": 0.4090909090909091, "Indonesian,English,Spanish,Malay": 0.4375, "Indonesian,English,Spanish,Vietnamese": 0.3181818181818182, "Indonesian,English,Chinese,Malay": 0.375, "Indonesian,English,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,English,Malay,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Spanish,Chinese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Malay": 0.2840909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Chinese,Malay": 0.25, "Indonesian,Filipino,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay": 0.4034090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.30113636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.3465909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.2897727272727273, "English,Filipino,Spanish,Chinese": 0.23295454545454544, "English,Filipino,Spanish,Malay": 0.23295454545454544, "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "English,Filipino,Chinese,Malay": 0.20454545454545456, "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, "English,Filipino,Malay,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese,Malay": 0.36363636363636365, "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, "English,Spanish,Malay,Vietnamese": 0.30113636363636365, "English,Chinese,Malay,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.21022727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.21022727272727273, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, "Indonesian,English,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.3465909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2727272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, "AC3_2": 0.4922957874512534, "AC3_3": 0.40647856403651084, "AC3_4": 0.33730220161128927, "AC3_5": 0.27847440694132897, "AC3_6": 0.22691837490206343, "AC3_7": 0.18082282677570657 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6116504854368932 }, "prompt_2": { "accuracy": 0.6116504854368932 }, "prompt_3": { "accuracy": 0.6116504854368932 }, "prompt_4": { "accuracy": 0.5728155339805825 }, "prompt_5": { "accuracy": 0.6310679611650486 } }, "cn_eval": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.38095238095238093 }, "prompt_3": { "accuracy": 0.41904761904761906 }, "prompt_4": { "accuracy": 0.3904761904761905 }, "prompt_5": { "accuracy": 0.44761904761904764 } }, "us_eval": { "prompt_1": { "accuracy": 0.6261682242990654 }, "prompt_2": { "accuracy": 0.6261682242990654 }, "prompt_3": { "accuracy": 0.6448598130841121 }, "prompt_4": { "accuracy": 0.6542056074766355 }, "prompt_5": { "accuracy": 0.6542056074766355 } }, "ph_eval": { "prompt_1": { "accuracy": 0.38, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.2, "history": 0.2, "literature": 0.2, "politics": 0.6, "culture": 0.8, "film": 0.4, "law": 0.2, "geography": 0.7 } }, "prompt_2": { "accuracy": 0.4, "category_acc": { "brand": 0.4, "demographics": 0.2, "biology": 0.4, "history": 0.2, "literature": 0.2, "politics": 0.6, "culture": 0.6, "film": 0.5, "law": 0.2, "geography": 0.7 } }, "prompt_3": { "accuracy": 0.37, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.5, "culture": 0.6, "film": 0.3, "law": 0.2, "geography": 0.7 } }, "prompt_4": { "accuracy": 0.39, "category_acc": { "brand": 0.3, "demographics": 0.0, "biology": 0.2, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.6, "culture": 0.6, "film": 0.5, "law": 0.3, "geography": 0.7 } }, "prompt_5": { "accuracy": 0.38, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.2, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.6, "culture": 0.6, "film": 0.4, "law": 0.3, "geography": 0.7 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.2574017356444316 }, "prompt_2": { "bleu_score": 0.2697397453044214 }, "prompt_3": { "bleu_score": 0.2710806888883188 }, "prompt_4": { "bleu_score": 0.252810003719897 }, "prompt_5": { "bleu_score": 0.223438870981906 } }, "indommlu": { "prompt_1": { "accuracy": 0.49495961012083584, "category_acc": { "History": 0.4819277108433735, "Geography": 0.46938775510204084, "Lampungic": 0.3877551020408163, "Social science": 0.7328881469115192, "Balinese": 0.34182590233545646, "Makassarese": 0.34946236559139787, "Banjarese": 0.4027777777777778, "Chemistry": 0.32116788321167883, "Biology": 0.43431952662721895, "Science": 0.6171310629514963, "Christian religion": 0.5920398009950248, "Art": 0.6039933444259568, "Islam religion": 0.6017069701280228, "Hindu religion": 0.5066666666666667, "Madurese": 0.3152542372881356, "Sport": 0.5337837837837838, "Indonesian language": 0.5709838107098381, "Physics": 0.4202020202020202, "Minangkabau culture": 0.39195979899497485, "Dayak language": 0.26605504587155965, "Sociology": 0.4879032258064516, "Economy": 0.4733606557377049, "Sundanese": 0.3777009507346586, "Javanese": 0.3538306451612903, "Civic education": 0.5951359084406295 } }, "prompt_2": { "accuracy": 0.4968956539154817, "category_acc": { "History": 0.4799196787148594, "Geography": 0.47346938775510206, "Lampungic": 0.3469387755102041, "Social science": 0.7445742904841403, "Balinese": 0.3375796178343949, "Makassarese": 0.3602150537634409, "Banjarese": 0.3680555555555556, "Chemistry": 0.32262773722627736, "Biology": 0.44260355029585796, "Science": 0.6243550051599587, "Christian religion": 0.6119402985074627, "Art": 0.6173044925124792, "Islam religion": 0.6273115220483642, "Hindu religion": 0.52, "Madurese": 0.3152542372881356, "Sport": 0.527027027027027, "Indonesian language": 0.5628891656288917, "Physics": 0.40404040404040403, "Minangkabau culture": 0.3768844221105528, "Dayak language": 0.27522935779816515, "Sociology": 0.48185483870967744, "Economy": 0.4651639344262295, "Sundanese": 0.38547968885047534, "Javanese": 0.3719758064516129, "Civic education": 0.597997138769671 } }, "prompt_3": { "accuracy": 0.4931570865878897, "category_acc": { "History": 0.4718875502008032, "Geography": 0.463265306122449, "Lampungic": 0.3401360544217687, "Social science": 0.7345575959933222, "Balinese": 0.31210191082802546, "Makassarese": 0.3548387096774194, "Banjarese": 0.4166666666666667, "Chemistry": 0.31532846715328466, "Biology": 0.44023668639053254, "Science": 0.6181630546955624, "Christian religion": 0.6019900497512438, "Art": 0.610648918469218, "Islam religion": 0.6216216216216216, "Hindu religion": 0.5133333333333333, "Madurese": 0.3050847457627119, "Sport": 0.527027027027027, "Indonesian language": 0.562266500622665, "Physics": 0.4121212121212121, "Minangkabau culture": 0.40703517587939697, "Dayak language": 0.3211009174311927, "Sociology": 0.4798387096774194, "Economy": 0.45901639344262296, "Sundanese": 0.3742437337942956, "Javanese": 0.37701612903225806, "Civic education": 0.586552217453505 } }, "prompt_4": { "accuracy": 0.49355764737298885, "category_acc": { "History": 0.4738955823293173, "Geography": 0.45510204081632655, "Lampungic": 0.35374149659863946, "Social science": 0.7245409015025042, "Balinese": 0.34182590233545646, "Makassarese": 0.3763440860215054, "Banjarese": 0.3888888888888889, "Chemistry": 0.3167883211678832, "Biology": 0.4461538461538462, "Science": 0.6160990712074303, "Christian religion": 0.6069651741293532, "Art": 0.6056572379367721, "Islam religion": 0.6116642958748222, "Hindu religion": 0.5, "Madurese": 0.2847457627118644, "Sport": 0.5337837837837838, "Indonesian language": 0.5675591531755916, "Physics": 0.402020202020202, "Minangkabau culture": 0.38190954773869346, "Dayak language": 0.27522935779816515, "Sociology": 0.49193548387096775, "Economy": 0.4672131147540984, "Sundanese": 0.3777009507346586, "Javanese": 0.3629032258064516, "Civic education": 0.5994277539341917 } }, "prompt_5": { "accuracy": 0.4947593297282863, "category_acc": { "History": 0.4779116465863454, "Geography": 0.4448979591836735, "Lampungic": 0.3469387755102041, "Social science": 0.7128547579298832, "Balinese": 0.3333333333333333, "Makassarese": 0.3709677419354839, "Banjarese": 0.4097222222222222, "Chemistry": 0.3284671532846715, "Biology": 0.44260355029585796, "Science": 0.6202270381836945, "Christian religion": 0.6268656716417911, "Art": 0.6023294509151415, "Islam religion": 0.6031294452347084, "Hindu religion": 0.5, "Madurese": 0.3152542372881356, "Sport": 0.5540540540540541, "Indonesian language": 0.5691158156911582, "Physics": 0.4161616161616162, "Minangkabau culture": 0.3969849246231156, "Dayak language": 0.28440366972477066, "Sociology": 0.4717741935483871, "Economy": 0.4672131147540984, "Sundanese": 0.3802938634399309, "Javanese": 0.37399193548387094, "Civic education": 0.5908440629470673 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.2904210030825523 }, "prompt_2": { "bleu_score": 0.36955498730132713 }, "prompt_3": { "bleu_score": 0.36547048717499736 }, "prompt_4": { "bleu_score": 0.3656155033164473 }, "prompt_5": { "bleu_score": 0.3613834087838442 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.1361309104530537 }, "prompt_2": { "bleu_score": 0.22571437892643548 }, "prompt_3": { "bleu_score": 0.2207638338470559 }, "prompt_4": { "bleu_score": 0.21907103091395946 }, "prompt_5": { "bleu_score": 0.18624794146447873 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.2141645930096053 }, "prompt_2": { "bleu_score": 0.22386319490546705 }, "prompt_3": { "bleu_score": 0.22159896503568985 }, "prompt_4": { "bleu_score": 0.22119680625556593 }, "prompt_5": { "bleu_score": 0.20898856132141055 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.20246195784882284 }, "prompt_2": { "bleu_score": 0.34339277822910913 }, "prompt_3": { "bleu_score": 0.3414705878211933 }, "prompt_4": { "bleu_score": 0.3421444160411124 }, "prompt_5": { "bleu_score": 0.31648254812909576 } }, "mmlu": { "prompt_1": { "accuracy": 0.5472578763127188 }, "prompt_2": { "accuracy": 0.5694282380396732 }, "prompt_3": { "accuracy": 0.5530921820303384 }, "prompt_4": { "accuracy": 0.558926487747958 }, "prompt_5": { "accuracy": 0.5320886814469078 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.5574544154451198, "category_acc": { "high_school_european_history": 0.7073170731707317, "business_ethics": 0.6161616161616161, "clinical_knowledge": 0.6287878787878788, "medical_genetics": 0.6565656565656566, "high_school_us_history": 0.7438423645320197, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.7288135593220338, "virology": 0.4727272727272727, "high_school_microeconomics": 0.5780590717299579, "econometrics": 0.336283185840708, "college_computer_science": 0.5656565656565656, "high_school_biology": 0.6699029126213593, "abstract_algebra": 0.35353535353535354, "professional_accounting": 0.4128113879003559, "philosophy": 0.5838709677419355, "professional_medicine": 0.6568265682656826, "nutrition": 0.5836065573770491, "global_facts": 0.35353535353535354, "machine_learning": 0.3963963963963964, "security_studies": 0.5819672131147541, "public_relations": 0.6513761467889908, "professional_psychology": 0.5319148936170213, "prehistory": 0.5820433436532507, "anatomy": 0.5447761194029851, "human_sexuality": 0.6461538461538462, "college_medicine": 0.6162790697674418, "high_school_government_and_politics": 0.8020833333333334, "college_chemistry": 0.3939393939393939, "logical_fallacies": 0.7037037037037037, "high_school_geography": 0.7258883248730964, "elementary_mathematics": 0.3793103448275862, "human_aging": 0.6396396396396397, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.78125, "formal_logic": 0.48, "high_school_statistics": 0.40930232558139534, "international_law": 0.7166666666666667, "high_school_mathematics": 0.31226765799256506, "high_school_computer_science": 0.5757575757575758, "conceptual_physics": 0.45726495726495725, "miscellaneous": 0.7851662404092071, "high_school_chemistry": 0.43564356435643564, "marketing": 0.8454935622317596, "professional_law": 0.42204827136333983, "management": 0.7647058823529411, "college_physics": 0.36633663366336633, "jurisprudence": 0.616822429906542, "world_religions": 0.8411764705882353, "sociology": 0.76, "us_foreign_policy": 0.7474747474747475, "high_school_macroeconomics": 0.5347043701799485, "computer_security": 0.7272727272727273, "moral_scenarios": 0.2550335570469799, "moral_disputes": 0.6231884057971014, "electrical_engineering": 0.5138888888888888, "astronomy": 0.5761589403973509, "college_biology": 0.6573426573426573 } }, "prompt_2": { "accuracy": 0.560672148730783, "category_acc": { "high_school_european_history": 0.7195121951219512, "business_ethics": 0.5555555555555556, "clinical_knowledge": 0.6553030303030303, "medical_genetics": 0.6262626262626263, "high_school_us_history": 0.7536945812807881, "high_school_physics": 0.3333333333333333, "high_school_world_history": 0.7372881355932204, "virology": 0.4727272727272727, "high_school_microeconomics": 0.6075949367088608, "econometrics": 0.3805309734513274, "college_computer_science": 0.5454545454545454, "high_school_biology": 0.6731391585760518, "abstract_algebra": 0.31313131313131315, "professional_accounting": 0.4199288256227758, "philosophy": 0.6096774193548387, "professional_medicine": 0.6346863468634686, "nutrition": 0.5737704918032787, "global_facts": 0.36363636363636365, "machine_learning": 0.4144144144144144, "security_studies": 0.6311475409836066, "public_relations": 0.6697247706422018, "professional_psychology": 0.5515548281505729, "prehistory": 0.6099071207430341, "anatomy": 0.5223880597014925, "human_sexuality": 0.6384615384615384, "college_medicine": 0.5872093023255814, "high_school_government_and_politics": 0.796875, "college_chemistry": 0.42424242424242425, "logical_fallacies": 0.6975308641975309, "high_school_geography": 0.751269035532995, "elementary_mathematics": 0.3819628647214854, "human_aging": 0.6351351351351351, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.7830882352941176, "formal_logic": 0.464, "high_school_statistics": 0.4418604651162791, "international_law": 0.7, "high_school_mathematics": 0.2825278810408922, "high_school_computer_science": 0.5959595959595959, "conceptual_physics": 0.48717948717948717, "miscellaneous": 0.7902813299232737, "high_school_chemistry": 0.4405940594059406, "marketing": 0.8669527896995708, "professional_law": 0.395955642530985, "management": 0.7647058823529411, "college_physics": 0.4158415841584158, "jurisprudence": 0.6448598130841121, "world_religions": 0.8176470588235294, "sociology": 0.77, "us_foreign_policy": 0.7777777777777778, "high_school_macroeconomics": 0.5449871465295629, "computer_security": 0.7171717171717171, "moral_scenarios": 0.25279642058165547, "moral_disputes": 0.6434782608695652, "electrical_engineering": 0.5416666666666666, "astronomy": 0.5629139072847682, "college_biology": 0.6643356643356644 } }, "prompt_3": { "accuracy": 0.5483732570611369, "category_acc": { "high_school_european_history": 0.6951219512195121, "business_ethics": 0.6161616161616161, "clinical_knowledge": 0.625, "medical_genetics": 0.6464646464646465, "high_school_us_history": 0.7487684729064039, "high_school_physics": 0.34, "high_school_world_history": 0.711864406779661, "virology": 0.48484848484848486, "high_school_microeconomics": 0.6118143459915611, "econometrics": 0.35398230088495575, "college_computer_science": 0.5050505050505051, "high_school_biology": 0.656957928802589, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.40569395017793597, "philosophy": 0.6225806451612903, "professional_medicine": 0.6088560885608856, "nutrition": 0.5606557377049181, "global_facts": 0.35353535353535354, "machine_learning": 0.38738738738738737, "security_studies": 0.6311475409836066, "public_relations": 0.6513761467889908, "professional_psychology": 0.5368248772504092, "prehistory": 0.6037151702786377, "anatomy": 0.5074626865671642, "human_sexuality": 0.6076923076923076, "college_medicine": 0.5581395348837209, "high_school_government_and_politics": 0.7760416666666666, "college_chemistry": 0.41414141414141414, "logical_fallacies": 0.654320987654321, "high_school_geography": 0.7157360406091371, "elementary_mathematics": 0.3713527851458886, "human_aging": 0.6396396396396397, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.7573529411764706, "formal_logic": 0.48, "high_school_statistics": 0.3953488372093023, "international_law": 0.7, "high_school_mathematics": 0.3048327137546468, "high_school_computer_science": 0.5252525252525253, "conceptual_physics": 0.44871794871794873, "miscellaneous": 0.7749360613810742, "high_school_chemistry": 0.42574257425742573, "marketing": 0.8154506437768241, "professional_law": 0.4116112198303979, "management": 0.7450980392156863, "college_physics": 0.36633663366336633, "jurisprudence": 0.6448598130841121, "world_religions": 0.8058823529411765, "sociology": 0.73, "us_foreign_policy": 0.7272727272727273, "high_school_macroeconomics": 0.4961439588688946, "computer_security": 0.7070707070707071, "moral_scenarios": 0.25615212527964204, "moral_disputes": 0.6202898550724638, "electrical_engineering": 0.5277777777777778, "astronomy": 0.5629139072847682, "college_biology": 0.6363636363636364 } }, "prompt_4": { "accuracy": 0.542652842331069, "category_acc": { "high_school_european_history": 0.7195121951219512, "business_ethics": 0.5555555555555556, "clinical_knowledge": 0.571969696969697, "medical_genetics": 0.6262626262626263, "high_school_us_history": 0.7586206896551724, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.7203389830508474, "virology": 0.48484848484848486, "high_school_microeconomics": 0.5780590717299579, "econometrics": 0.3274336283185841, "college_computer_science": 0.5454545454545454, "high_school_biology": 0.656957928802589, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.4128113879003559, "philosophy": 0.567741935483871, "professional_medicine": 0.5867158671586716, "nutrition": 0.5967213114754099, "global_facts": 0.36363636363636365, "machine_learning": 0.36936936936936937, "security_studies": 0.5983606557377049, "public_relations": 0.6513761467889908, "professional_psychology": 0.5122749590834698, "prehistory": 0.6006191950464397, "anatomy": 0.48507462686567165, "human_sexuality": 0.6538461538461539, "college_medicine": 0.5581395348837209, "high_school_government_and_politics": 0.7864583333333334, "college_chemistry": 0.42424242424242425, "logical_fallacies": 0.6790123456790124, "high_school_geography": 0.7360406091370558, "elementary_mathematics": 0.3713527851458886, "human_aging": 0.6486486486486487, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.7536764705882353, "formal_logic": 0.48, "high_school_statistics": 0.4186046511627907, "international_law": 0.6916666666666667, "high_school_mathematics": 0.30111524163568776, "high_school_computer_science": 0.5555555555555556, "conceptual_physics": 0.44871794871794873, "miscellaneous": 0.7595907928388747, "high_school_chemistry": 0.42574257425742573, "marketing": 0.8025751072961373, "professional_law": 0.4044357469015003, "management": 0.7549019607843137, "college_physics": 0.37623762376237624, "jurisprudence": 0.616822429906542, "world_religions": 0.8235294117647058, "sociology": 0.735, "us_foreign_policy": 0.7272727272727273, "high_school_macroeconomics": 0.4961439588688946, "computer_security": 0.6767676767676768, "moral_scenarios": 0.2550335570469799, "moral_disputes": 0.5884057971014492, "electrical_engineering": 0.5208333333333334, "astronomy": 0.5496688741721855, "college_biology": 0.6433566433566433 } }, "prompt_5": { "accuracy": 0.5278512692170182, "category_acc": { "high_school_european_history": 0.6829268292682927, "business_ethics": 0.5555555555555556, "clinical_knowledge": 0.5795454545454546, "medical_genetics": 0.5252525252525253, "high_school_us_history": 0.7438423645320197, "high_school_physics": 0.28, "high_school_world_history": 0.7033898305084746, "virology": 0.44242424242424244, "high_school_microeconomics": 0.5738396624472574, "econometrics": 0.37168141592920356, "college_computer_science": 0.5151515151515151, "high_school_biology": 0.6310679611650486, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.4306049822064057, "philosophy": 0.5612903225806452, "professional_medicine": 0.5756457564575646, "nutrition": 0.5442622950819672, "global_facts": 0.35353535353535354, "machine_learning": 0.3963963963963964, "security_studies": 0.5901639344262295, "public_relations": 0.6238532110091743, "professional_psychology": 0.5008183306055647, "prehistory": 0.5944272445820433, "anatomy": 0.5, "human_sexuality": 0.6307692307692307, "college_medicine": 0.5116279069767442, "high_school_government_and_politics": 0.7604166666666666, "college_chemistry": 0.40404040404040403, "logical_fallacies": 0.6296296296296297, "high_school_geography": 0.7208121827411168, "elementary_mathematics": 0.38992042440318303, "human_aging": 0.6126126126126126, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.7113970588235294, "formal_logic": 0.464, "high_school_statistics": 0.4232558139534884, "international_law": 0.6583333333333333, "high_school_mathematics": 0.30855018587360594, "high_school_computer_science": 0.5454545454545454, "conceptual_physics": 0.46153846153846156, "miscellaneous": 0.7544757033248082, "high_school_chemistry": 0.42574257425742573, "marketing": 0.7725321888412017, "professional_law": 0.4050880626223092, "management": 0.7058823529411765, "college_physics": 0.3564356435643564, "jurisprudence": 0.6261682242990654, "world_religions": 0.7823529411764706, "sociology": 0.73, "us_foreign_policy": 0.6565656565656566, "high_school_macroeconomics": 0.4987146529562982, "computer_security": 0.6767676767676768, "moral_scenarios": 0.22930648769574943, "moral_disputes": 0.5652173913043478, "electrical_engineering": 0.5069444444444444, "astronomy": 0.48344370860927155, "college_biology": 0.5594405594405595 } } }, "c_eval": { "prompt_1": { "accuracy": 0.4227340267459138 }, "prompt_2": { "accuracy": 0.4279346210995542 }, "prompt_3": { "accuracy": 0.4323922734026746 }, "prompt_4": { "accuracy": 0.40936106983655274 }, "prompt_5": { "accuracy": 0.4323922734026746 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.4246575342465753, "category_acc": { "computer_network": 0.5, "operating_system": 0.5416666666666666, "computer_architecture": 0.4230769230769231, "college_programming": 0.42857142857142855, "college_physics": 0.4166666666666667, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.375, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.47619047619047616, "electrical_engineer": 0.35714285714285715, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.25, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.4230769230769231, "middle_school_physics": 0.625, "middle_school_chemistry": 0.56, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.45, "business_administration": 0.4473684210526316, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.6206896551724138, "education_science": 0.47058823529411764, "teacher_qualification": 0.6122448979591837, "high_school_politics": 0.375, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.5384615384615384, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.75, "logic": 0.48148148148148145, "law": 0.3793103448275862, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.5, "legal_professional": 0.5, "high_school_chinese": 0.375, "high_school_history": 0.56, "middle_school_history": 0.6296296296296297, "civil_servant": 0.36538461538461536, "sports_science": 0.375, "plant_protection": 0.4444444444444444, "basic_medicine": 0.4583333333333333, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.45098039215686275, "accountant": 0.37037037037037035, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.4166666666666667, "tax_accountant": 0.2962962962962963, "physician": 0.3888888888888889 } }, "prompt_2": { "accuracy": 0.4265255292652553, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5833333333333334, "computer_architecture": 0.4230769230769231, "college_programming": 0.42857142857142855, "college_physics": 0.375, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.42857142857142855, "electrical_engineer": 0.35714285714285715, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.25, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.625, "middle_school_chemistry": 0.52, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.4166666666666667, "business_administration": 0.42105263157894735, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.6896551724137931, "education_science": 0.47058823529411764, "teacher_qualification": 0.6326530612244898, "high_school_politics": 0.375, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.6538461538461539, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.5185185185185185, "law": 0.3793103448275862, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.5294117647058824, "legal_professional": 0.5357142857142857, "high_school_chinese": 0.375, "high_school_history": 0.52, "middle_school_history": 0.6666666666666666, "civil_servant": 0.34615384615384615, "sports_science": 0.25, "plant_protection": 0.4444444444444444, "basic_medicine": 0.5, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.43137254901960786, "accountant": 0.37037037037037035, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.4444444444444444, "tax_accountant": 0.2962962962962963, "physician": 0.42592592592592593 } }, "prompt_3": { "accuracy": 0.4339975093399751, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5833333333333334, "computer_architecture": 0.4230769230769231, "college_programming": 0.47619047619047616, "college_physics": 0.375, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.375, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.47619047619047616, "electrical_engineer": 0.38095238095238093, "metrology_engineer": 0.4482758620689655, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.25, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.25, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.6666666666666666, "middle_school_chemistry": 0.52, "veterinary_medicine": 0.42857142857142855, "college_economics": 0.45, "business_administration": 0.42105263157894735, "marxism": 0.5, "mao_zedong_thought": 0.6206896551724138, "education_science": 0.5, "teacher_qualification": 0.6122448979591837, "high_school_politics": 0.4166666666666667, "high_school_geography": 0.375, "middle_school_politics": 0.5, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.7916666666666666, "logic": 0.5185185185185185, "law": 0.41379310344827586, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.5588235294117647, "legal_professional": 0.5, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.52, "middle_school_history": 0.6296296296296297, "civil_servant": 0.40384615384615385, "sports_science": 0.25, "plant_protection": 0.5185185185185185, "basic_medicine": 0.5, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.4117647058823529, "accountant": 0.3888888888888889, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.5, "tax_accountant": 0.25925925925925924, "physician": 0.4444444444444444 } }, "prompt_4": { "accuracy": 0.4178082191780822, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.5833333333333334, "computer_architecture": 0.4230769230769231, "college_programming": 0.38095238095238093, "college_physics": 0.375, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.25, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.4782608695652174, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.4230769230769231, "middle_school_physics": 0.625, "middle_school_chemistry": 0.6, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.36666666666666664, "business_administration": 0.42105263157894735, "marxism": 0.375, "mao_zedong_thought": 0.6896551724137931, "education_science": 0.5, "teacher_qualification": 0.5102040816326531, "high_school_politics": 0.375, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.5384615384615384, "middle_school_geography": 0.5294117647058824, "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.625, "logic": 0.48148148148148145, "law": 0.3448275862068966, "chinese_language_and_literature": 0.42857142857142855, "art_studies": 0.39473684210526316, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.5, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.48, "middle_school_history": 0.6296296296296297, "civil_servant": 0.34615384615384615, "sports_science": 0.375, "plant_protection": 0.4444444444444444, "basic_medicine": 0.5, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.4117647058823529, "accountant": 0.37037037037037035, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.35185185185185186, "physician": 0.42592592592592593 } }, "prompt_5": { "accuracy": 0.4259028642590286, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.5416666666666666, "computer_architecture": 0.4230769230769231, "college_programming": 0.42857142857142855, "college_physics": 0.4166666666666667, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.375, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.3333333333333333, "metrology_engineer": 0.4482758620689655, "high_school_mathematics": 0.43478260869565216, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.38461538461538464, "middle_school_physics": 0.625, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.48333333333333334, "business_administration": 0.42105263157894735, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.6896551724137931, "education_science": 0.5, "teacher_qualification": 0.5714285714285714, "high_school_politics": 0.4583333333333333, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.5, "middle_school_geography": 0.5294117647058824, "modern_chinese_history": 0.39285714285714285, "ideological_and_moral_cultivation": 0.7916666666666666, "logic": 0.48148148148148145, "law": 0.3103448275862069, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.39473684210526316, "professional_tour_guide": 0.47058823529411764, "legal_professional": 0.5357142857142857, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.52, "middle_school_history": 0.6296296296296297, "civil_servant": 0.36538461538461536, "sports_science": 0.2916666666666667, "plant_protection": 0.5555555555555556, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.43137254901960786, "accountant": 0.35185185185185186, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.4444444444444444, "tax_accountant": 0.2962962962962963, "physician": 0.4444444444444444 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.4157706093189964 }, "prompt_2": { "accuracy": 0.3978494623655914 }, "prompt_3": { "accuracy": 0.4121863799283154 }, "prompt_4": { "accuracy": 0.4014336917562724 }, "prompt_5": { "accuracy": 0.3942652329749104 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.41504058021067175, "category_acc": { "agronomy": 0.34911242603550297, "anatomy": 0.22297297297297297, "ancient_chinese": 0.25609756097560976, "arts": 0.43125, "astronomy": 0.3090909090909091, "business_ethics": 0.46411483253588515, "chinese_civil_service_exam": 0.3375, "chinese_driving_rule": 0.6183206106870229, "chinese_food_culture": 0.3161764705882353, "chinese_foreign_policy": 0.48598130841121495, "chinese_history": 0.43653250773993807, "chinese_literature": 0.3088235294117647, "chinese_teacher_qualification": 0.4860335195530726, "clinical_knowledge": 0.34177215189873417, "college_actuarial_science": 0.2830188679245283, "college_education": 0.4766355140186916, "college_engineering_hydrology": 0.4528301886792453, "college_law": 0.3888888888888889, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.36792452830188677, "college_medicine": 0.38095238095238093, "computer_science": 0.47058823529411764, "computer_security": 0.5263157894736842, "conceptual_physics": 0.4421768707482993, "construction_project_management": 0.3669064748201439, "economics": 0.4968553459119497, "education": 0.44785276073619634, "electrical_engineering": 0.42441860465116277, "elementary_chinese": 0.30158730158730157, "elementary_commonsense": 0.37373737373737376, "elementary_information_and_technology": 0.6176470588235294, "elementary_mathematics": 0.30434782608695654, "ethnology": 0.4888888888888889, "food_science": 0.45454545454545453, "genetics": 0.3409090909090909, "global_facts": 0.4563758389261745, "high_school_biology": 0.34911242603550297, "high_school_chemistry": 0.32575757575757575, "high_school_geography": 0.4152542372881356, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.3090909090909091, "high_school_politics": 0.4195804195804196, "human_sexuality": 0.4365079365079365, "international_law": 0.3783783783783784, "journalism": 0.47674418604651164, "jurisprudence": 0.49148418491484186, "legal_and_moral_basis": 0.7757009345794392, "logical": 0.37398373983739835, "machine_learning": 0.3114754098360656, "management": 0.5190476190476191, "marketing": 0.5666666666666667, "marxist_theory": 0.48148148148148145, "modern_chinese": 0.3620689655172414, "nutrition": 0.3793103448275862, "philosophy": 0.47619047619047616, "professional_accounting": 0.4342857142857143, "professional_law": 0.3459715639810427, "professional_medicine": 0.28191489361702127, "professional_psychology": 0.4827586206896552, "public_relations": 0.5574712643678161, "security_study": 0.45925925925925926, "sociology": 0.504424778761062, "sports_science": 0.41818181818181815, "traditional_chinese_medicine": 0.3027027027027027, "virology": 0.39644970414201186, "world_history": 0.391304347826087, "world_religions": 0.41875 } }, "prompt_2": { "accuracy": 0.41875323778276635, "category_acc": { "agronomy": 0.33727810650887574, "anatomy": 0.22297297297297297, "ancient_chinese": 0.2804878048780488, "arts": 0.4125, "astronomy": 0.3090909090909091, "business_ethics": 0.44019138755980863, "chinese_civil_service_exam": 0.33125, "chinese_driving_rule": 0.6335877862595419, "chinese_food_culture": 0.3088235294117647, "chinese_foreign_policy": 0.4766355140186916, "chinese_history": 0.4086687306501548, "chinese_literature": 0.3137254901960784, "chinese_teacher_qualification": 0.4972067039106145, "clinical_knowledge": 0.3459915611814346, "college_actuarial_science": 0.25471698113207547, "college_education": 0.5233644859813084, "college_engineering_hydrology": 0.4811320754716981, "college_law": 0.37037037037037035, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.3490566037735849, "college_medicine": 0.34798534798534797, "computer_science": 0.4803921568627451, "computer_security": 0.4853801169590643, "conceptual_physics": 0.4421768707482993, "construction_project_management": 0.38848920863309355, "economics": 0.4779874213836478, "education": 0.48466257668711654, "electrical_engineering": 0.43023255813953487, "elementary_chinese": 0.30158730158730157, "elementary_commonsense": 0.3888888888888889, "elementary_information_and_technology": 0.6428571428571429, "elementary_mathematics": 0.30434782608695654, "ethnology": 0.4962962962962963, "food_science": 0.46153846153846156, "genetics": 0.3693181818181818, "global_facts": 0.4429530201342282, "high_school_biology": 0.35502958579881655, "high_school_chemistry": 0.29545454545454547, "high_school_geography": 0.3983050847457627, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.43356643356643354, "human_sexuality": 0.4444444444444444, "international_law": 0.3837837837837838, "journalism": 0.5232558139534884, "jurisprudence": 0.49391727493917276, "legal_and_moral_basis": 0.7757009345794392, "logical": 0.3983739837398374, "machine_learning": 0.36065573770491804, "management": 0.5428571428571428, "marketing": 0.5555555555555556, "marxist_theory": 0.4708994708994709, "modern_chinese": 0.35344827586206895, "nutrition": 0.42758620689655175, "philosophy": 0.45714285714285713, "professional_accounting": 0.44571428571428573, "professional_law": 0.33175355450236965, "professional_medicine": 0.3058510638297872, "professional_psychology": 0.4698275862068966, "public_relations": 0.5517241379310345, "security_study": 0.5185185185185185, "sociology": 0.4823008849557522, "sports_science": 0.41818181818181815, "traditional_chinese_medicine": 0.3081081081081081, "virology": 0.4319526627218935, "world_history": 0.40993788819875776, "world_religions": 0.43125 } }, "prompt_3": { "accuracy": 0.41745812467622173, "category_acc": { "agronomy": 0.3254437869822485, "anatomy": 0.20270270270270271, "ancient_chinese": 0.2621951219512195, "arts": 0.41875, "astronomy": 0.2545454545454545, "business_ethics": 0.45933014354066987, "chinese_civil_service_exam": 0.325, "chinese_driving_rule": 0.5954198473282443, "chinese_food_culture": 0.3161764705882353, "chinese_foreign_policy": 0.48598130841121495, "chinese_history": 0.3931888544891641, "chinese_literature": 0.3137254901960784, "chinese_teacher_qualification": 0.5027932960893855, "clinical_knowledge": 0.34177215189873417, "college_actuarial_science": 0.3018867924528302, "college_education": 0.5233644859813084, "college_engineering_hydrology": 0.49056603773584906, "college_law": 0.37037037037037035, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.3490566037735849, "college_medicine": 0.34798534798534797, "computer_science": 0.49019607843137253, "computer_security": 0.49707602339181284, "conceptual_physics": 0.41496598639455784, "construction_project_management": 0.381294964028777, "economics": 0.5031446540880503, "education": 0.49693251533742333, "electrical_engineering": 0.436046511627907, "elementary_chinese": 0.30158730158730157, "elementary_commonsense": 0.3888888888888889, "elementary_information_and_technology": 0.6596638655462185, "elementary_mathematics": 0.3, "ethnology": 0.4740740740740741, "food_science": 0.46153846153846156, "genetics": 0.375, "global_facts": 0.42953020134228187, "high_school_biology": 0.3609467455621302, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.423728813559322, "high_school_mathematics": 0.27439024390243905, "high_school_physics": 0.34545454545454546, "high_school_politics": 0.4195804195804196, "human_sexuality": 0.46825396825396826, "international_law": 0.3837837837837838, "journalism": 0.4883720930232558, "jurisprudence": 0.48418491484184917, "legal_and_moral_basis": 0.7757009345794392, "logical": 0.3821138211382114, "machine_learning": 0.36885245901639346, "management": 0.5142857142857142, "marketing": 0.5555555555555556, "marxist_theory": 0.4656084656084656, "modern_chinese": 0.3620689655172414, "nutrition": 0.4206896551724138, "philosophy": 0.4380952380952381, "professional_accounting": 0.42857142857142855, "professional_law": 0.3459715639810427, "professional_medicine": 0.324468085106383, "professional_psychology": 0.47844827586206895, "public_relations": 0.5517241379310345, "security_study": 0.5111111111111111, "sociology": 0.4823008849557522, "sports_science": 0.4, "traditional_chinese_medicine": 0.34054054054054056, "virology": 0.4319526627218935, "world_history": 0.40372670807453415, "world_religions": 0.41875 } }, "prompt_4": { "accuracy": 0.40986012778449316, "category_acc": { "agronomy": 0.33136094674556216, "anatomy": 0.22297297297297297, "ancient_chinese": 0.24390243902439024, "arts": 0.4, "astronomy": 0.3090909090909091, "business_ethics": 0.45933014354066987, "chinese_civil_service_exam": 0.36875, "chinese_driving_rule": 0.6106870229007634, "chinese_food_culture": 0.3014705882352941, "chinese_foreign_policy": 0.45794392523364486, "chinese_history": 0.4117647058823529, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.5139664804469274, "clinical_knowledge": 0.350210970464135, "college_actuarial_science": 0.24528301886792453, "college_education": 0.514018691588785, "college_engineering_hydrology": 0.44339622641509435, "college_law": 0.39814814814814814, "college_mathematics": 0.3333333333333333, "college_medical_statistics": 0.39622641509433965, "college_medicine": 0.31868131868131866, "computer_science": 0.49019607843137253, "computer_security": 0.5087719298245614, "conceptual_physics": 0.41496598639455784, "construction_project_management": 0.381294964028777, "economics": 0.48427672955974843, "education": 0.48466257668711654, "electrical_engineering": 0.4127906976744186, "elementary_chinese": 0.2857142857142857, "elementary_commonsense": 0.37373737373737376, "elementary_information_and_technology": 0.6386554621848739, "elementary_mathematics": 0.2782608695652174, "ethnology": 0.4888888888888889, "food_science": 0.44755244755244755, "genetics": 0.3465909090909091, "global_facts": 0.4228187919463087, "high_school_biology": 0.33727810650887574, "high_school_chemistry": 0.3409090909090909, "high_school_geography": 0.3898305084745763, "high_school_mathematics": 0.23170731707317074, "high_school_physics": 0.3181818181818182, "high_school_politics": 0.45454545454545453, "human_sexuality": 0.42857142857142855, "international_law": 0.3621621621621622, "journalism": 0.47674418604651164, "jurisprudence": 0.44768856447688565, "legal_and_moral_basis": 0.7523364485981309, "logical": 0.37398373983739835, "machine_learning": 0.3524590163934426, "management": 0.5142857142857142, "marketing": 0.55, "marxist_theory": 0.48148148148148145, "modern_chinese": 0.3879310344827586, "nutrition": 0.43448275862068964, "philosophy": 0.44761904761904764, "professional_accounting": 0.41714285714285715, "professional_law": 0.35071090047393366, "professional_medicine": 0.2872340425531915, "professional_psychology": 0.47844827586206895, "public_relations": 0.5344827586206896, "security_study": 0.45925925925925926, "sociology": 0.49557522123893805, "sports_science": 0.41818181818181815, "traditional_chinese_medicine": 0.31351351351351353, "virology": 0.40236686390532544, "world_history": 0.37888198757763975, "world_religions": 0.4 } }, "prompt_5": { "accuracy": 0.41875323778276635, "category_acc": { "agronomy": 0.33136094674556216, "anatomy": 0.24324324324324326, "ancient_chinese": 0.2804878048780488, "arts": 0.4125, "astronomy": 0.296969696969697, "business_ethics": 0.47368421052631576, "chinese_civil_service_exam": 0.35625, "chinese_driving_rule": 0.6259541984732825, "chinese_food_culture": 0.3235294117647059, "chinese_foreign_policy": 0.4953271028037383, "chinese_history": 0.43962848297213625, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.5083798882681564, "clinical_knowledge": 0.35864978902953587, "college_actuarial_science": 0.24528301886792453, "college_education": 0.5420560747663551, "college_engineering_hydrology": 0.4811320754716981, "college_law": 0.35185185185185186, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.41509433962264153, "college_medicine": 0.3772893772893773, "computer_science": 0.47058823529411764, "computer_security": 0.5029239766081871, "conceptual_physics": 0.4421768707482993, "construction_project_management": 0.381294964028777, "economics": 0.48427672955974843, "education": 0.44785276073619634, "electrical_engineering": 0.4127906976744186, "elementary_chinese": 0.30158730158730157, "elementary_commonsense": 0.40404040404040403, "elementary_information_and_technology": 0.6596638655462185, "elementary_mathematics": 0.3347826086956522, "ethnology": 0.5037037037037037, "food_science": 0.45454545454545453, "genetics": 0.35795454545454547, "global_facts": 0.44966442953020136, "high_school_biology": 0.35502958579881655, "high_school_chemistry": 0.25, "high_school_geography": 0.4152542372881356, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.32727272727272727, "high_school_politics": 0.4195804195804196, "human_sexuality": 0.4444444444444444, "international_law": 0.3837837837837838, "journalism": 0.5058139534883721, "jurisprudence": 0.48905109489051096, "legal_and_moral_basis": 0.7616822429906542, "logical": 0.3902439024390244, "machine_learning": 0.36885245901639346, "management": 0.5190476190476191, "marketing": 0.5388888888888889, "marxist_theory": 0.48148148148148145, "modern_chinese": 0.3793103448275862, "nutrition": 0.36551724137931035, "philosophy": 0.47619047619047616, "professional_accounting": 0.44, "professional_law": 0.35071090047393366, "professional_medicine": 0.3058510638297872, "professional_psychology": 0.4698275862068966, "public_relations": 0.5574712643678161, "security_study": 0.4666666666666667, "sociology": 0.48672566371681414, "sports_science": 0.41818181818181815, "traditional_chinese_medicine": 0.31351351351351353, "virology": 0.40828402366863903, "world_history": 0.391304347826087, "world_religions": 0.4 } } }, "zbench": { "prompt_1": { "accuracy": 0.30303030303030304 }, "prompt_2": { "accuracy": 0.3939393939393939 }, "prompt_3": { "accuracy": 0.30303030303030304 }, "prompt_4": { "accuracy": 0.3333333333333333 }, "prompt_5": { "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.7090909090909091 }, "prompt_2": { "accuracy": 0.6545454545454545 }, "prompt_3": { "accuracy": 0.6613636363636364 }, "prompt_4": { "accuracy": 0.6909090909090909 }, "prompt_5": { "accuracy": 0.6818181818181818 } }, "ocnli": { "prompt_1": { "accuracy": 0.4105084745762712 }, "prompt_2": { "accuracy": 0.42915254237288136 }, "prompt_3": { "accuracy": 0.4105084745762712 }, "prompt_4": { "accuracy": 0.41966101694915253 }, "prompt_5": { "accuracy": 0.408135593220339 } }, "c3": { "prompt_1": { "accuracy": 0.7857142857142857 }, "prompt_2": { "accuracy": 0.7756170531039641 }, "prompt_3": { "accuracy": 0.7729992520568437 }, "prompt_4": { "accuracy": 0.7789827973074046 }, "prompt_5": { "accuracy": 0.7756170531039641 } }, "dream": { "prompt_1": { "accuracy": 0.8574228319451249 }, "prompt_2": { "accuracy": 0.849093581577658 }, "prompt_3": { "accuracy": 0.8417442430181283 }, "prompt_4": { "accuracy": 0.8564429201371877 }, "prompt_5": { "accuracy": 0.854483096521313 } }, "samsum": { "prompt_1": { "rouge1": 0.337934193303483, "rouge2": 0.13154283775211653, "rougeL": 0.2614570898741684, "avg_rouge": 0.24364470697658933 }, "prompt_2": { "rouge1": 0.370426617540352, "rouge2": 0.14460103670266045, "rougeL": 0.2857311872813845, "avg_rouge": 0.2669196138414656 }, "prompt_3": { "rouge1": 0.3526840141075432, "rouge2": 0.13318361789311367, "rougeL": 0.27212472005604, "avg_rouge": 0.2526641173522323 }, "prompt_4": { "rouge1": 0.34499225565885405, "rouge2": 0.131082047202366, "rougeL": 0.26542039826170644, "avg_rouge": 0.24716490037430883 }, "prompt_5": { "rouge1": 0.36434660312828493, "rouge2": 0.1342337732311142, "rougeL": 0.2812806022953663, "avg_rouge": 0.2599536595515885 } }, "dialogsum": { "prompt_1": { "rouge1": 0.23171463702568543, "rouge2": 0.06482582508272836, "rougeL": 0.17113911650235575, "avg_rouge": 0.15589319287025652 }, "prompt_2": { "rouge1": 0.23232726794331263, "rouge2": 0.06413743246594411, "rougeL": 0.17150409279989268, "avg_rouge": 0.15598959773638313 }, "prompt_3": { "rouge1": 0.23218474445664966, "rouge2": 0.06416276534806573, "rougeL": 0.17133699125529367, "avg_rouge": 0.15589483368666968 }, "prompt_4": { "rouge1": 0.23040880796958316, "rouge2": 0.06366423137632587, "rougeL": 0.16981402032766788, "avg_rouge": 0.1546290198911923 }, "prompt_5": { "rouge1": 0.23125728609443794, "rouge2": 0.0640530946503918, "rougeL": 0.17086375161470452, "avg_rouge": 0.15539137745317808 } }, "sst2": { "prompt_1": { "accuracy": 0.9151376146788991 }, "prompt_2": { "accuracy": 0.9002293577981652 }, "prompt_3": { "accuracy": 0.9139908256880734 }, "prompt_4": { "accuracy": 0.9094036697247706 }, "prompt_5": { "accuracy": 0.9220183486238532 } }, "cola": { "prompt_1": { "accuracy": 0.7804410354745925 }, "prompt_2": { "accuracy": 0.7813998082454459 }, "prompt_3": { "accuracy": 0.7746883988494727 }, "prompt_4": { "accuracy": 0.7813998082454459 }, "prompt_5": { "accuracy": 0.7785234899328859 } }, "qqp": { "prompt_1": { "accuracy": 0.6935 }, "prompt_2": { "accuracy": 0.6935 }, "prompt_3": { "accuracy": 0.6855 }, "prompt_4": { "accuracy": 0.6905 }, "prompt_5": { "accuracy": 0.688 } }, "mnli": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.4 }, "prompt_5": { "accuracy": 0.7 } }, "qnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.6 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.5 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "meta_llama_3_8b_instruct": { "model_size": "8B", "model_link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9378151260504202, "language_acc": { "Spanish": 0.9394957983193277, "English": 0.9478991596638655, "Chinese": 0.9294117647058824, "Vietnamese": 0.934453781512605 }, "consistency_score_2": 0.9329131652661066, "consistency_score_3": 0.901890756302521, "consistency_score_4": 0.8798319327731092, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9445378151260504, "Spanish,Chinese": 0.926890756302521, "Spanish,Vietnamese": 0.9302521008403362, "English,Chinese": 0.9336134453781513, "English,Vietnamese": 0.9428571428571428, "Chinese,Vietnamese": 0.9193277310924369 }, "3_combine": { "Spanish,English,Chinese": 0.9042016806722689, "Spanish,English,Vietnamese": 0.9117647058823529, "Spanish,Chinese,Vietnamese": 0.8915966386554622, "English,Chinese,Vietnamese": 0.9 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8798319327731092 } }, "AC3_2": 0.9353577231845365, "AC3_3": 0.9195021893149736, "AC3_4": 0.9078986934080167 }, "prompt_2": { "overall_acc": 0.9336134453781513, "language_acc": { "Spanish": 0.9310924369747899, "English": 0.9470588235294117, "Chinese": 0.9252100840336135, "Vietnamese": 0.9310924369747899 }, "consistency_score_2": 0.922689075630252, "consistency_score_3": 0.8876050420168067, "consistency_score_4": 0.865546218487395, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9361344537815126, "Spanish,Chinese": 0.9168067226890756, "Spanish,Vietnamese": 0.9310924369747899, "English,Chinese": 0.9218487394957983, "English,Vietnamese": 0.9277310924369748, "Chinese,Vietnamese": 0.9025210084033614 }, "3_combine": { "Spanish,English,Chinese": 0.8890756302521008, "Spanish,English,Vietnamese": 0.9, "Spanish,Chinese,Vietnamese": 0.880672268907563, "English,Chinese,Vietnamese": 0.880672268907563 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.865546218487395 } }, "AC3_2": 0.9281191154096758, "AC3_3": 0.9100281016278929, "AC3_4": 0.8982922453863125 }, "prompt_3": { "overall_acc": 0.9218487394957983, "language_acc": { "Spanish": 0.9193277310924369, "English": 0.934453781512605, "Chinese": 0.9126050420168067, "Vietnamese": 0.9210084033613445 }, "consistency_score_2": 0.903781512605042, "consistency_score_3": 0.859873949579832, "consistency_score_4": 0.8285714285714286, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9218487394957983, "Spanish,Chinese": 0.8949579831932774, "Spanish,Vietnamese": 0.9067226890756303, "English,Chinese": 0.9008403361344538, "English,Vietnamese": 0.9159663865546218, "Chinese,Vietnamese": 0.8823529411764706 }, "3_combine": { "Spanish,English,Chinese": 0.8621848739495799, "Spanish,English,Vietnamese": 0.8756302521008403, "Spanish,Chinese,Vietnamese": 0.8478991596638655, "English,Chinese,Vietnamese": 0.853781512605042 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8285714285714286 } }, "AC3_2": 0.9127257254530945, "AC3_3": 0.8897834902828757, "AC3_4": 0.8727247787718962 }, "prompt_4": { "overall_acc": 0.9304621848739496, "language_acc": { "Spanish": 0.9260504201680673, "English": 0.9436974789915966, "Chinese": 0.9193277310924369, "Vietnamese": 0.9327731092436975 }, "consistency_score_2": 0.9184873949579831, "consistency_score_3": 0.8808823529411764, "consistency_score_4": 0.8546218487394958, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9252100840336135, "Spanish,Chinese": 0.9109243697478991, "Spanish,Vietnamese": 0.9226890756302522, "English,Chinese": 0.9168067226890756, "English,Vietnamese": 0.9277310924369748, "Chinese,Vietnamese": 0.907563025210084 }, "3_combine": { "Spanish,English,Chinese": 0.8798319327731092, "Spanish,English,Vietnamese": 0.8907563025210085, "Spanish,Chinese,Vietnamese": 0.8739495798319328, "English,Chinese,Vietnamese": 0.8789915966386554 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8546218487394958 } }, "AC3_2": 0.9244360122824432, "AC3_3": 0.904993723257036, "AC3_4": 0.8909309563034448 }, "prompt_5": { "overall_acc": 0.8934873949579832, "language_acc": { "Spanish": 0.9058823529411765, "English": 0.915126050420168, "Chinese": 0.8697478991596639, "Vietnamese": 0.8831932773109243 }, "consistency_score_2": 0.8563025210084034, "consistency_score_3": 0.7915966386554621, "consistency_score_4": 0.746218487394958, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8966386554621849, "Spanish,Chinese": 0.8436974789915966, "Spanish,Vietnamese": 0.8495798319327731, "English,Chinese": 0.8554621848739495, "English,Vietnamese": 0.8672268907563025, "Chinese,Vietnamese": 0.8252100840336134 }, "3_combine": { "Spanish,English,Chinese": 0.8050420168067227, "Spanish,English,Vietnamese": 0.8142857142857143, "Spanish,Chinese,Vietnamese": 0.7647058823529411, "English,Chinese,Vietnamese": 0.7823529411764706 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.746218487394958 } }, "AC3_2": 0.8744998491152861, "AC3_3": 0.8394615394593914, "AC3_4": 0.8132395199736593 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.6076190476190476, "language_acc": { "Filipino": 0.5466666666666666, "Vietnamese": 0.5933333333333334, "Chinese": 0.64, "Spanish": 0.6133333333333333, "Malay": 0.6066666666666667, "Indonesian": 0.52, "English": 0.7333333333333333 }, "consistency_score_2": 0.5895238095238096, "consistency_score_3": 0.431047619047619, "consistency_score_4": 0.34742857142857136, "consistency_score_5": 0.29365079365079366, "consistency_score_6": 0.2552380952380952, "consistency_score_7": 0.22666666666666666, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5266666666666666, "Filipino,Chinese": 0.5933333333333334, "Filipino,Spanish": 0.5333333333333333, "Filipino,Malay": 0.5133333333333333, "Filipino,Indonesian": 0.4866666666666667, "Filipino,English": 0.58, "Vietnamese,Chinese": 0.66, "Vietnamese,Spanish": 0.58, "Vietnamese,Malay": 0.6, "Vietnamese,Indonesian": 0.6333333333333333, "Vietnamese,English": 0.5933333333333334, "Chinese,Spanish": 0.5533333333333333, "Chinese,Malay": 0.6466666666666666, "Chinese,Indonesian": 0.5933333333333334, "Chinese,English": 0.64, "Spanish,Malay": 0.5533333333333333, "Spanish,Indonesian": 0.5733333333333334, "Spanish,English": 0.6733333333333333, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.6466666666666666, "Indonesian,English": 0.5533333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.43333333333333335, "Filipino,Vietnamese,Spanish": 0.38, "Filipino,Vietnamese,Malay": 0.38666666666666666, "Filipino,Vietnamese,Indonesian": 0.38, "Filipino,Vietnamese,English": 0.3933333333333333, "Filipino,Chinese,Spanish": 0.38, "Filipino,Chinese,Malay": 0.43333333333333335, "Filipino,Chinese,Indonesian": 0.38, "Filipino,Chinese,English": 0.4533333333333333, "Filipino,Spanish,Malay": 0.36666666666666664, "Filipino,Spanish,Indonesian": 0.35333333333333333, "Filipino,Spanish,English": 0.4266666666666667, "Filipino,Malay,Indonesian": 0.38, "Filipino,Malay,English": 0.42, "Filipino,Indonesian,English": 0.36666666666666664, "Vietnamese,Chinese,Spanish": 0.44666666666666666, "Vietnamese,Chinese,Malay": 0.48, "Vietnamese,Chinese,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,English": 0.4866666666666667, "Vietnamese,Spanish,Malay": 0.42, "Vietnamese,Spanish,Indonesian": 0.4533333333333333, "Vietnamese,Spanish,English": 0.44666666666666666, "Vietnamese,Malay,Indonesian": 0.48, "Vietnamese,Malay,English": 0.46, "Vietnamese,Indonesian,English": 0.44, "Chinese,Spanish,Malay": 0.44666666666666666, "Chinese,Spanish,Indonesian": 0.4066666666666667, "Chinese,Spanish,English": 0.4666666666666667, "Chinese,Malay,Indonesian": 0.4866666666666667, "Chinese,Malay,English": 0.5, "Chinese,Indonesian,English": 0.44, "Spanish,Malay,Indonesian": 0.44, "Spanish,Malay,English": 0.48, "Spanish,Indonesian,English": 0.44, "Malay,Indonesian,English": 0.4666666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Malay": 0.3466666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Chinese,English": 0.34, "Filipino,Vietnamese,Spanish,Malay": 0.30666666666666664, "Filipino,Vietnamese,Spanish,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Spanish,English": 0.32666666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,Malay,English": 0.32666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.31333333333333335, "Filipino,Chinese,Spanish,Malay": 0.32, "Filipino,Chinese,Spanish,Indonesian": 0.2866666666666667, "Filipino,Chinese,Spanish,English": 0.3333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.32666666666666666, "Filipino,Chinese,Malay,English": 0.37333333333333335, "Filipino,Chinese,Indonesian,English": 0.31333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, "Filipino,Spanish,Malay,English": 0.34, "Filipino,Spanish,Indonesian,English": 0.3, "Filipino,Malay,Indonesian,English": 0.31333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Indonesian": 0.36666666666666664, "Vietnamese,Chinese,Spanish,English": 0.38, "Vietnamese,Chinese,Malay,Indonesian": 0.3933333333333333, "Vietnamese,Chinese,Malay,English": 0.4066666666666667, "Vietnamese,Chinese,Indonesian,English": 0.38666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Spanish,Malay,English": 0.36666666666666664, "Vietnamese,Spanish,Indonesian,English": 0.36, "Vietnamese,Malay,Indonesian,English": 0.38666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.36666666666666664, "Chinese,Spanish,Malay,English": 0.4, "Chinese,Spanish,Indonesian,English": 0.36, "Chinese,Malay,Indonesian,English": 0.38666666666666666, "Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.3, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,Malay,English": 0.28, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.28, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.3, "Filipino,Chinese,Spanish,Indonesian,English": 0.26, "Filipino,Chinese,Malay,Indonesian,English": 0.28, "Filipino,Spanish,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.32, "Vietnamese,Chinese,Malay,Indonesian,English": 0.34, "Vietnamese,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 } }, "AC3_2": 0.5984346705565198, "AC3_3": 0.5043249237592222, "AC3_4": 0.44208103021184547, "AC3_5": 0.39594760055319295, "AC3_6": 0.3594744034062524, "AC3_7": 0.3301674276620994 }, "prompt_2": { "overall_acc": 0.5771428571428573, "language_acc": { "Filipino": 0.52, "Vietnamese": 0.56, "Chinese": 0.6266666666666667, "Spanish": 0.5533333333333333, "Malay": 0.5666666666666667, "Indonesian": 0.5466666666666666, "English": 0.6666666666666666 }, "consistency_score_2": 0.5517460317460318, "consistency_score_3": 0.3826666666666666, "consistency_score_4": 0.30019047619047623, "consistency_score_5": 0.2526984126984127, "consistency_score_6": 0.22190476190476188, "consistency_score_7": 0.2, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5266666666666666, "Filipino,Chinese": 0.5, "Filipino,Spanish": 0.5133333333333333, "Filipino,Malay": 0.49333333333333335, "Filipino,Indonesian": 0.49333333333333335, "Filipino,English": 0.5333333333333333, "Vietnamese,Chinese": 0.56, "Vietnamese,Spanish": 0.5666666666666667, "Vietnamese,Malay": 0.52, "Vietnamese,Indonesian": 0.54, "Vietnamese,English": 0.6133333333333333, "Chinese,Spanish": 0.5666666666666667, "Chinese,Malay": 0.5266666666666666, "Chinese,Indonesian": 0.56, "Chinese,English": 0.5666666666666667, "Spanish,Malay": 0.5333333333333333, "Spanish,Indonesian": 0.5133333333333333, "Spanish,English": 0.6666666666666666, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.6, "Indonesian,English": 0.5466666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.36666666666666664, "Filipino,Vietnamese,Spanish": 0.36, "Filipino,Vietnamese,Malay": 0.35333333333333333, "Filipino,Vietnamese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,English": 0.37333333333333335, "Filipino,Chinese,Spanish": 0.3333333333333333, "Filipino,Chinese,Malay": 0.32666666666666666, "Filipino,Chinese,Indonesian": 0.34, "Filipino,Chinese,English": 0.36, "Filipino,Spanish,Malay": 0.3333333333333333, "Filipino,Spanish,Indonesian": 0.3, "Filipino,Spanish,English": 0.3933333333333333, "Filipino,Malay,Indonesian": 0.36666666666666664, "Filipino,Malay,English": 0.36, "Filipino,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Chinese,Malay": 0.36666666666666664, "Vietnamese,Chinese,Indonesian": 0.38666666666666666, "Vietnamese,Chinese,English": 0.41333333333333333, "Vietnamese,Spanish,Malay": 0.38666666666666666, "Vietnamese,Spanish,Indonesian": 0.3933333333333333, "Vietnamese,Spanish,English": 0.4533333333333333, "Vietnamese,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Malay,English": 0.42, "Vietnamese,Indonesian,English": 0.4266666666666667, "Chinese,Spanish,Malay": 0.38, "Chinese,Spanish,Indonesian": 0.36666666666666664, "Chinese,Spanish,English": 0.44, "Chinese,Malay,Indonesian": 0.4066666666666667, "Chinese,Malay,English": 0.4, "Chinese,Indonesian,English": 0.4, "Spanish,Malay,Indonesian": 0.4066666666666667, "Spanish,Malay,English": 0.43333333333333335, "Spanish,Indonesian,English": 0.4, "Malay,Indonesian,English": 0.44 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.28, "Filipino,Vietnamese,Chinese,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Chinese,English": 0.28, "Filipino,Vietnamese,Spanish,Malay": 0.28, "Filipino,Vietnamese,Spanish,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Spanish,English": 0.30666666666666664, "Filipino,Vietnamese,Malay,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Malay,English": 0.29333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.29333333333333333, "Filipino,Chinese,Spanish,Malay": 0.2733333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.26, "Filipino,Chinese,Spanish,English": 0.2866666666666667, "Filipino,Chinese,Malay,Indonesian": 0.2733333333333333, "Filipino,Chinese,Malay,English": 0.28, "Filipino,Chinese,Indonesian,English": 0.29333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Spanish,Malay,English": 0.2866666666666667, "Filipino,Spanish,Indonesian,English": 0.26, "Filipino,Malay,Indonesian,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Indonesian": 0.32, "Vietnamese,Chinese,Spanish,English": 0.35333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.31333333333333335, "Vietnamese,Chinese,Malay,English": 0.31333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.32, "Vietnamese,Spanish,Malay,English": 0.34, "Vietnamese,Spanish,Indonesian,English": 0.34, "Vietnamese,Malay,Indonesian,English": 0.3466666666666667, "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, "Chinese,Spanish,Malay,English": 0.3333333333333333, "Chinese,Spanish,Indonesian,English": 0.32, "Chinese,Malay,Indonesian,English": 0.32666666666666666, "Spanish,Malay,Indonesian,English": 0.34 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.24, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.24, "Filipino,Chinese,Malay,Indonesian,English": 0.24, "Filipino,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.28, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.28, "Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2 } }, "AC3_2": 0.5641587658185775, "AC3_3": 0.4602024210673534, "AC3_4": 0.3949531666319877, "AC3_5": 0.35149633835528205, "AC3_6": 0.3205584879557959, "AC3_7": 0.2970588234911873 }, "prompt_3": { "overall_acc": 0.5466666666666666, "language_acc": { "Filipino": 0.5066666666666667, "Vietnamese": 0.5466666666666666, "Chinese": 0.56, "Spanish": 0.52, "Malay": 0.49333333333333335, "Indonesian": 0.5466666666666666, "English": 0.6533333333333333 }, "consistency_score_2": 0.5396825396825398, "consistency_score_3": 0.3613333333333334, "consistency_score_4": 0.26647619047619053, "consistency_score_5": 0.20952380952380953, "consistency_score_6": 0.17238095238095238, "consistency_score_7": 0.14666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5133333333333333, "Filipino,Chinese": 0.5133333333333333, "Filipino,Spanish": 0.5, "Filipino,Malay": 0.48, "Filipino,Indonesian": 0.5533333333333333, "Filipino,English": 0.52, "Vietnamese,Chinese": 0.6066666666666667, "Vietnamese,Spanish": 0.54, "Vietnamese,Malay": 0.5733333333333334, "Vietnamese,Indonesian": 0.54, "Vietnamese,English": 0.58, "Chinese,Spanish": 0.5733333333333334, "Chinese,Malay": 0.48, "Chinese,Indonesian": 0.44, "Chinese,English": 0.5533333333333333, "Spanish,Malay": 0.5733333333333334, "Spanish,Indonesian": 0.5133333333333333, "Spanish,English": 0.5933333333333334, "Malay,Indonesian": 0.5533333333333333, "Malay,English": 0.5466666666666666, "Indonesian,English": 0.5866666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.38, "Filipino,Vietnamese,Spanish": 0.34, "Filipino,Vietnamese,Malay": 0.3466666666666667, "Filipino,Vietnamese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,English": 0.36, "Filipino,Chinese,Spanish": 0.35333333333333333, "Filipino,Chinese,Malay": 0.3, "Filipino,Chinese,Indonesian": 0.30666666666666664, "Filipino,Chinese,English": 0.3466666666666667, "Filipino,Spanish,Malay": 0.3333333333333333, "Filipino,Spanish,Indonesian": 0.3466666666666667, "Filipino,Spanish,English": 0.35333333333333333, "Filipino,Malay,Indonesian": 0.3466666666666667, "Filipino,Malay,English": 0.3333333333333333, "Filipino,Indonesian,English": 0.36, "Vietnamese,Chinese,Spanish": 0.41333333333333333, "Vietnamese,Chinese,Malay": 0.37333333333333335, "Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,English": 0.4066666666666667, "Vietnamese,Spanish,Malay": 0.4, "Vietnamese,Spanish,Indonesian": 0.35333333333333333, "Vietnamese,Spanish,English": 0.38666666666666666, "Vietnamese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Malay,English": 0.4, "Vietnamese,Indonesian,English": 0.41333333333333333, "Chinese,Spanish,Malay": 0.37333333333333335, "Chinese,Spanish,Indonesian": 0.32, "Chinese,Spanish,English": 0.38666666666666666, "Chinese,Malay,Indonesian": 0.3, "Chinese,Malay,English": 0.3333333333333333, "Chinese,Indonesian,English": 0.3466666666666667, "Spanish,Malay,Indonesian": 0.37333333333333335, "Spanish,Malay,English": 0.38666666666666666, "Spanish,Indonesian,English": 0.38666666666666666, "Malay,Indonesian,English": 0.3933333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.26, "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.26, "Filipino,Vietnamese,Spanish,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Spanish,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Malay,English": 0.2733333333333333, "Filipino,Vietnamese,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay": 0.25333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Chinese,Spanish,English": 0.26666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.22, "Filipino,Chinese,Malay,English": 0.23333333333333334, "Filipino,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.26, "Filipino,Spanish,Malay,English": 0.24, "Filipino,Spanish,Indonesian,English": 0.26, "Filipino,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay": 0.31333333333333335, "Vietnamese,Chinese,Spanish,Indonesian": 0.2733333333333333, "Vietnamese,Chinese,Spanish,English": 0.30666666666666664, "Vietnamese,Chinese,Malay,Indonesian": 0.26, "Vietnamese,Chinese,Malay,English": 0.2866666666666667, "Vietnamese,Chinese,Indonesian,English": 0.29333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.28, "Vietnamese,Spanish,Malay,English": 0.3, "Vietnamese,Spanish,Indonesian,English": 0.2866666666666667, "Vietnamese,Malay,Indonesian,English": 0.3, "Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, "Chinese,Spanish,Malay,English": 0.26, "Chinese,Spanish,Indonesian,English": 0.26, "Chinese,Malay,Indonesian,English": 0.24, "Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,English": 0.22, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2, "Filipino,Chinese,Spanish,Malay,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.24, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.22, "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 } }, "AC3_2": 0.5431521526871927, "AC3_3": 0.4350856582976534, "AC3_4": 0.3582978058433421, "AC3_5": 0.3029387069288725, "AC3_6": 0.2621103752394857, "AC3_7": 0.23128205124869328 }, "prompt_4": { "overall_acc": 0.602857142857143, "language_acc": { "Filipino": 0.5666666666666667, "Vietnamese": 0.5866666666666667, "Chinese": 0.64, "Spanish": 0.6066666666666667, "Malay": 0.5866666666666667, "Indonesian": 0.5466666666666666, "English": 0.6866666666666666 }, "consistency_score_2": 0.5930158730158731, "consistency_score_3": 0.43466666666666665, "consistency_score_4": 0.3512380952380953, "consistency_score_5": 0.299047619047619, "consistency_score_6": 0.2619047619047619, "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5466666666666666, "Filipino,Chinese": 0.6066666666666667, "Filipino,Spanish": 0.5733333333333334, "Filipino,Malay": 0.5733333333333334, "Filipino,Indonesian": 0.58, "Filipino,English": 0.58, "Vietnamese,Chinese": 0.6133333333333333, "Vietnamese,Spanish": 0.6333333333333333, "Vietnamese,Malay": 0.5666666666666667, "Vietnamese,Indonesian": 0.56, "Vietnamese,English": 0.5933333333333334, "Chinese,Spanish": 0.5866666666666667, "Chinese,Malay": 0.5933333333333334, "Chinese,Indonesian": 0.5733333333333334, "Chinese,English": 0.6133333333333333, "Spanish,Malay": 0.5933333333333334, "Spanish,Indonesian": 0.56, "Spanish,English": 0.66, "Malay,Indonesian": 0.6666666666666666, "Malay,English": 0.6133333333333333, "Indonesian,English": 0.5666666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.4266666666666667, "Filipino,Vietnamese,Spanish": 0.43333333333333335, "Filipino,Vietnamese,Malay": 0.4066666666666667, "Filipino,Vietnamese,Indonesian": 0.4, "Filipino,Vietnamese,English": 0.4066666666666667, "Filipino,Chinese,Spanish": 0.43333333333333335, "Filipino,Chinese,Malay": 0.4266666666666667, "Filipino,Chinese,Indonesian": 0.41333333333333333, "Filipino,Chinese,English": 0.43333333333333335, "Filipino,Spanish,Malay": 0.44, "Filipino,Spanish,Indonesian": 0.41333333333333333, "Filipino,Spanish,English": 0.44666666666666666, "Filipino,Malay,Indonesian": 0.44, "Filipino,Malay,English": 0.43333333333333335, "Filipino,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Spanish": 0.4533333333333333, "Vietnamese,Chinese,Malay": 0.43333333333333335, "Vietnamese,Chinese,Indonesian": 0.42, "Vietnamese,Chinese,English": 0.44, "Vietnamese,Spanish,Malay": 0.44666666666666666, "Vietnamese,Spanish,Indonesian": 0.43333333333333335, "Vietnamese,Spanish,English": 0.4866666666666667, "Vietnamese,Malay,Indonesian": 0.4533333333333333, "Vietnamese,Malay,English": 0.43333333333333335, "Vietnamese,Indonesian,English": 0.42, "Chinese,Spanish,Malay": 0.4266666666666667, "Chinese,Spanish,Indonesian": 0.41333333333333333, "Chinese,Spanish,English": 0.4666666666666667, "Chinese,Malay,Indonesian": 0.4533333333333333, "Chinese,Malay,English": 0.44666666666666666, "Chinese,Indonesian,English": 0.41333333333333333, "Spanish,Malay,Indonesian": 0.4533333333333333, "Spanish,Malay,English": 0.47333333333333333, "Spanish,Indonesian,English": 0.4266666666666667, "Malay,Indonesian,English": 0.47333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.36, "Filipino,Vietnamese,Chinese,Malay": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,Chinese,English": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.37333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Spanish,English": 0.37333333333333335, "Filipino,Vietnamese,Malay,Indonesian": 0.34, "Filipino,Vietnamese,Malay,English": 0.34, "Filipino,Vietnamese,Indonesian,English": 0.32666666666666666, "Filipino,Chinese,Spanish,Malay": 0.36, "Filipino,Chinese,Spanish,Indonesian": 0.34, "Filipino,Chinese,Spanish,English": 0.35333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.34, "Filipino,Chinese,Malay,English": 0.3466666666666667, "Filipino,Chinese,Indonesian,English": 0.32, "Filipino,Spanish,Malay,Indonesian": 0.35333333333333333, "Filipino,Spanish,Malay,English": 0.38, "Filipino,Spanish,Indonesian,English": 0.3333333333333333, "Filipino,Malay,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish,Malay": 0.3466666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,Spanish,English": 0.38, "Vietnamese,Chinese,Malay,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,Malay,English": 0.3466666666666667, "Vietnamese,Chinese,Indonesian,English": 0.3466666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.36666666666666664, "Vietnamese,Spanish,Malay,English": 0.37333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.35333333333333333, "Vietnamese,Malay,Indonesian,English": 0.36, "Chinese,Spanish,Malay,Indonesian": 0.34, "Chinese,Spanish,Malay,English": 0.36666666666666664, "Chinese,Spanish,Indonesian,English": 0.34, "Chinese,Malay,Indonesian,English": 0.35333333333333333, "Spanish,Malay,Indonesian,English": 0.38666666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Spanish,English": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.28, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Spanish,Malay,English": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.30666666666666664, "Filipino,Chinese,Spanish,Indonesian,English": 0.28, "Filipino,Chinese,Malay,Indonesian,English": 0.28, "Filipino,Spanish,Malay,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3, "Vietnamese,Chinese,Spanish,Malay,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Malay,Indonesian,English": 0.29333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.31333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.3 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.26, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, "AC3_2": 0.5978960142091715, "AC3_3": 0.5051294289948243, "AC3_4": 0.4438684652873053, "AC3_5": 0.3997827726212339, "AC3_6": 0.3651667715122094, "AC3_7": 0.33644646920805593 }, "prompt_5": { "overall_acc": 0.5390476190476191, "language_acc": { "Filipino": 0.4266666666666667, "Vietnamese": 0.49333333333333335, "Chinese": 0.56, "Spanish": 0.6, "Malay": 0.5333333333333333, "Indonesian": 0.4866666666666667, "English": 0.6733333333333333 }, "consistency_score_2": 0.5266666666666666, "consistency_score_3": 0.3462857142857143, "consistency_score_4": 0.24952380952380956, "consistency_score_5": 0.1892063492063492, "consistency_score_6": 0.14857142857142858, "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4533333333333333, "Filipino,Chinese": 0.4666666666666667, "Filipino,Spanish": 0.47333333333333333, "Filipino,Malay": 0.44666666666666666, "Filipino,Indonesian": 0.43333333333333335, "Filipino,English": 0.47333333333333333, "Vietnamese,Chinese": 0.54, "Vietnamese,Spanish": 0.5333333333333333, "Vietnamese,Malay": 0.6133333333333333, "Vietnamese,Indonesian": 0.5333333333333333, "Vietnamese,English": 0.4866666666666667, "Chinese,Spanish": 0.5333333333333333, "Chinese,Malay": 0.5866666666666667, "Chinese,Indonesian": 0.5266666666666666, "Chinese,English": 0.5866666666666667, "Spanish,Malay": 0.5933333333333334, "Spanish,Indonesian": 0.5533333333333333, "Spanish,English": 0.5933333333333334, "Malay,Indonesian": 0.6133333333333333, "Malay,English": 0.5, "Indonesian,English": 0.52 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.30666666666666664, "Filipino,Vietnamese,Spanish": 0.3, "Filipino,Vietnamese,Malay": 0.32, "Filipino,Vietnamese,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,English": 0.2733333333333333, "Filipino,Chinese,Spanish": 0.30666666666666664, "Filipino,Chinese,Malay": 0.32, "Filipino,Chinese,Indonesian": 0.28, "Filipino,Chinese,English": 0.31333333333333335, "Filipino,Spanish,Malay": 0.32, "Filipino,Spanish,Indonesian": 0.3, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.2866666666666667, "Filipino,Malay,English": 0.2733333333333333, "Filipino,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish": 0.36666666666666664, "Vietnamese,Chinese,Malay": 0.4066666666666667, "Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,English": 0.36, "Vietnamese,Spanish,Malay": 0.44, "Vietnamese,Spanish,Indonesian": 0.38, "Vietnamese,Spanish,English": 0.34, "Vietnamese,Malay,Indonesian": 0.44, "Vietnamese,Malay,English": 0.36666666666666664, "Vietnamese,Indonesian,English": 0.35333333333333333, "Chinese,Spanish,Malay": 0.4066666666666667, "Chinese,Spanish,Indonesian": 0.36, "Chinese,Spanish,English": 0.38666666666666666, "Chinese,Malay,Indonesian": 0.41333333333333333, "Chinese,Malay,English": 0.38, "Chinese,Indonesian,English": 0.36666666666666664, "Spanish,Malay,Indonesian": 0.42, "Spanish,Malay,English": 0.38, "Spanish,Indonesian,English": 0.37333333333333335, "Malay,Indonesian,English": 0.36666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,English": 0.21333333333333335, "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Malay,English": 0.22, "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay": 0.24, "Filipino,Chinese,Spanish,Indonesian": 0.2, "Filipino,Chinese,Spanish,English": 0.23333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Malay,English": 0.21333333333333335, "Filipino,Chinese,Indonesian,English": 0.20666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.22, "Filipino,Spanish,Malay,English": 0.23333333333333334, "Filipino,Spanish,Indonesian,English": 0.21333333333333335, "Filipino,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Spanish,English": 0.26666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.32, "Vietnamese,Chinese,Malay,English": 0.29333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.26, "Vietnamese,Spanish,Malay,Indonesian": 0.3333333333333333, "Vietnamese,Spanish,Malay,English": 0.2866666666666667, "Vietnamese,Spanish,Indonesian,English": 0.26, "Vietnamese,Malay,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, "Chinese,Spanish,Malay,English": 0.2866666666666667, "Chinese,Spanish,Indonesian,English": 0.26, "Chinese,Malay,Indonesian,English": 0.2733333333333333, "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Spanish,English": 0.18, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.18, "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.22 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, "AC3_2": 0.532785224853194, "AC3_3": 0.42168182931061965, "AC3_4": 0.3411364158757646, "AC3_5": 0.2800979781240327, "AC3_6": 0.23294024531634128, "AC3_7": 0.1963005780048966 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.4935064935064935, "language_acc": { "Indonesian": 0.4715909090909091, "English": 0.5681818181818182, "Filipino": 0.4318181818181818, "Spanish": 0.5340909090909091, "Chinese": 0.5568181818181818, "Malay": 0.42045454545454547, "Vietnamese": 0.4715909090909091 }, "consistency_score_2": 0.6071428571428572, "consistency_score_3": 0.45, "consistency_score_4": 0.3626623376623376, "consistency_score_5": 0.3057359307359307, "consistency_score_6": 0.26461038961038963, "consistency_score_7": 0.23295454545454544, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5909090909090909, "Indonesian,Filipino": 0.5965909090909091, "Indonesian,Spanish": 0.6477272727272727, "Indonesian,Chinese": 0.6022727272727273, "Indonesian,Malay": 0.6647727272727273, "Indonesian,Vietnamese": 0.6477272727272727, "English,Filipino": 0.5454545454545454, "English,Spanish": 0.7840909090909091, "English,Chinese": 0.5965909090909091, "English,Malay": 0.5738636363636364, "English,Vietnamese": 0.7272727272727273, "Filipino,Spanish": 0.5738636363636364, "Filipino,Chinese": 0.4943181818181818, "Filipino,Malay": 0.6363636363636364, "Filipino,Vietnamese": 0.5284090909090909, "Spanish,Chinese": 0.5625, "Spanish,Malay": 0.6079545454545454, "Spanish,Vietnamese": 0.6477272727272727, "Chinese,Malay": 0.5397727272727273, "Chinese,Vietnamese": 0.6022727272727273, "Malay,Vietnamese": 0.5795454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.42045454545454547, "Indonesian,English,Spanish": 0.5340909090909091, "Indonesian,English,Chinese": 0.4431818181818182, "Indonesian,English,Malay": 0.45454545454545453, "Indonesian,English,Vietnamese": 0.5113636363636364, "Indonesian,Filipino,Spanish": 0.4602272727272727, "Indonesian,Filipino,Chinese": 0.375, "Indonesian,Filipino,Malay": 0.5, "Indonesian,Filipino,Vietnamese": 0.4318181818181818, "Indonesian,Spanish,Chinese": 0.4431818181818182, "Indonesian,Spanish,Malay": 0.4943181818181818, "Indonesian,Spanish,Vietnamese": 0.5, "Indonesian,Chinese,Malay": 0.4431818181818182, "Indonesian,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Spanish": 0.4772727272727273, "English,Filipino,Chinese": 0.3806818181818182, "English,Filipino,Malay": 0.42045454545454547, "English,Filipino,Vietnamese": 0.4431818181818182, "English,Spanish,Chinese": 0.4943181818181818, "English,Spanish,Malay": 0.5056818181818182, "English,Spanish,Vietnamese": 0.6022727272727273, "English,Chinese,Malay": 0.38636363636363635, "English,Chinese,Vietnamese": 0.48863636363636365, "English,Malay,Vietnamese": 0.4772727272727273, "Filipino,Spanish,Chinese": 0.3693181818181818, "Filipino,Spanish,Malay": 0.45454545454545453, "Filipino,Spanish,Vietnamese": 0.42045454545454547, "Filipino,Chinese,Malay": 0.375, "Filipino,Chinese,Vietnamese": 0.3693181818181818, "Filipino,Malay,Vietnamese": 0.4147727272727273, "Spanish,Chinese,Malay": 0.39204545454545453, "Spanish,Chinese,Vietnamese": 0.4431818181818182, "Spanish,Malay,Vietnamese": 0.45454545454545453, "Chinese,Malay,Vietnamese": 0.42045454545454547 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.38636363636363635, "Indonesian,English,Filipino,Chinese": 0.3181818181818182, "Indonesian,English,Filipino,Malay": 0.36363636363636365, "Indonesian,English,Filipino,Vietnamese": 0.3693181818181818, "Indonesian,English,Spanish,Chinese": 0.3977272727272727, "Indonesian,English,Spanish,Malay": 0.4147727272727273, "Indonesian,English,Spanish,Vietnamese": 0.4715909090909091, "Indonesian,English,Chinese,Malay": 0.3409090909090909, "Indonesian,English,Chinese,Vietnamese": 0.3977272727272727, "Indonesian,English,Malay,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.3125, "Indonesian,Filipino,Spanish,Malay": 0.3977272727272727, "Indonesian,Filipino,Spanish,Vietnamese": 0.35795454545454547, "Indonesian,Filipino,Chinese,Malay": 0.32386363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.32386363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.3693181818181818, "Indonesian,Spanish,Chinese,Malay": 0.3465909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.39204545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.39204545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.3693181818181818, "English,Filipino,Spanish,Chinese": 0.3352272727272727, "English,Filipino,Spanish,Malay": 0.3806818181818182, "English,Filipino,Spanish,Vietnamese": 0.3977272727272727, "English,Filipino,Chinese,Malay": 0.30113636363636365, "English,Filipino,Chinese,Vietnamese": 0.32386363636363635, "English,Filipino,Malay,Vietnamese": 0.36363636363636365, "English,Spanish,Chinese,Malay": 0.3522727272727273, "English,Spanish,Chinese,Vietnamese": 0.4147727272727273, "English,Spanish,Malay,Vietnamese": 0.42613636363636365, "English,Chinese,Malay,Vietnamese": 0.3465909090909091, "Filipino,Spanish,Chinese,Malay": 0.29545454545454547, "Filipino,Spanish,Chinese,Vietnamese": 0.3125, "Filipino,Spanish,Malay,Vietnamese": 0.3465909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.3068181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2897727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.3352272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.3409090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.2727272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,English,Filipino,Malay,Vietnamese": 0.32386363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.3125, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3693181818181818, "Indonesian,English,Spanish,Malay,Vietnamese": 0.375, "Indonesian,English,Chinese,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.3125, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818, "English,Filipino,Spanish,Chinese,Malay": 0.2727272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish,Malay,Vietnamese": 0.32954545454545453, "English,Filipino,Chinese,Malay,Vietnamese": 0.2727272727272727, "English,Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.25, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.30113636363636365, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.25, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.25 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544 } }, "AC3_2": 0.5444584913116792, "AC3_3": 0.4707501720079177, "AC3_4": 0.4180862746927325, "AC3_5": 0.3775642096090479, "AC3_6": 0.34450346223103956, "AC3_7": 0.31650584048461866 }, "prompt_2": { "overall_acc": 0.48701298701298706, "language_acc": { "Indonesian": 0.44886363636363635, "English": 0.5511363636363636, "Filipino": 0.4431818181818182, "Spanish": 0.5227272727272727, "Chinese": 0.5397727272727273, "Malay": 0.4772727272727273, "Vietnamese": 0.42613636363636365 }, "consistency_score_2": 0.5933441558441558, "consistency_score_3": 0.436525974025974, "consistency_score_4": 0.34902597402597413, "consistency_score_5": 0.2919372294372295, "consistency_score_6": 0.25162337662337664, "consistency_score_7": 0.2215909090909091, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5852272727272727, "Indonesian,Filipino": 0.5454545454545454, "Indonesian,Spanish": 0.6193181818181818, "Indonesian,Chinese": 0.5568181818181818, "Indonesian,Malay": 0.6590909090909091, "Indonesian,Vietnamese": 0.6079545454545454, "English,Filipino": 0.5284090909090909, "English,Spanish": 0.75, "English,Chinese": 0.5909090909090909, "English,Malay": 0.6306818181818182, "English,Vietnamese": 0.6079545454545454, "Filipino,Spanish": 0.5681818181818182, "Filipino,Chinese": 0.5284090909090909, "Filipino,Malay": 0.5738636363636364, "Filipino,Vietnamese": 0.5397727272727273, "Spanish,Chinese": 0.5681818181818182, "Spanish,Malay": 0.6647727272727273, "Spanish,Vietnamese": 0.6306818181818182, "Chinese,Malay": 0.5454545454545454, "Chinese,Vietnamese": 0.5909090909090909, "Malay,Vietnamese": 0.5681818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.3977272727272727, "Indonesian,English,Spanish": 0.5056818181818182, "Indonesian,English,Chinese": 0.42045454545454547, "Indonesian,English,Malay": 0.4943181818181818, "Indonesian,English,Vietnamese": 0.4375, "Indonesian,Filipino,Spanish": 0.4090909090909091, "Indonesian,Filipino,Chinese": 0.36363636363636365, "Indonesian,Filipino,Malay": 0.44886363636363635, "Indonesian,Filipino,Vietnamese": 0.39204545454545453, "Indonesian,Spanish,Chinese": 0.4147727272727273, "Indonesian,Spanish,Malay": 0.5170454545454546, "Indonesian,Spanish,Vietnamese": 0.45454545454545453, "Indonesian,Chinese,Malay": 0.4375, "Indonesian,Chinese,Vietnamese": 0.42045454545454547, "Indonesian,Malay,Vietnamese": 0.4715909090909091, "English,Filipino,Spanish": 0.4659090909090909, "English,Filipino,Chinese": 0.39204545454545453, "English,Filipino,Malay": 0.42613636363636365, "English,Filipino,Vietnamese": 0.38636363636363635, "English,Spanish,Chinese": 0.48863636363636365, "English,Spanish,Malay": 0.5454545454545454, "English,Spanish,Vietnamese": 0.5113636363636364, "English,Chinese,Malay": 0.4318181818181818, "English,Chinese,Vietnamese": 0.4318181818181818, "English,Malay,Vietnamese": 0.4431818181818182, "Filipino,Spanish,Chinese": 0.38636363636363635, "Filipino,Spanish,Malay": 0.44886363636363635, "Filipino,Spanish,Vietnamese": 0.4034090909090909, "Filipino,Chinese,Malay": 0.39204545454545453, "Filipino,Chinese,Vietnamese": 0.39204545454545453, "Filipino,Malay,Vietnamese": 0.39204545454545453, "Spanish,Chinese,Malay": 0.4375, "Spanish,Chinese,Vietnamese": 0.4318181818181818, "Spanish,Malay,Vietnamese": 0.4772727272727273, "Chinese,Malay,Vietnamese": 0.4090909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.3465909090909091, "Indonesian,English,Filipino,Chinese": 0.3125, "Indonesian,English,Filipino,Malay": 0.36363636363636365, "Indonesian,English,Filipino,Vietnamese": 0.3181818181818182, "Indonesian,English,Spanish,Chinese": 0.3693181818181818, "Indonesian,English,Spanish,Malay": 0.4375, "Indonesian,English,Spanish,Vietnamese": 0.39204545454545453, "Indonesian,English,Chinese,Malay": 0.36363636363636365, "Indonesian,English,Chinese,Vietnamese": 0.3409090909090909, "Indonesian,English,Malay,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.2897727272727273, "Indonesian,Filipino,Spanish,Malay": 0.36363636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.3181818181818182, "Indonesian,Filipino,Chinese,Malay": 0.3181818181818182, "Indonesian,Filipino,Chinese,Vietnamese": 0.3125, "Indonesian,Filipino,Malay,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Chinese,Malay": 0.35795454545454547, "Indonesian,Spanish,Chinese,Vietnamese": 0.3409090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.3977272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.3409090909090909, "English,Filipino,Spanish,Chinese": 0.3465909090909091, "English,Filipino,Spanish,Malay": 0.375, "English,Filipino,Spanish,Vietnamese": 0.3465909090909091, "English,Filipino,Chinese,Malay": 0.3352272727272727, "English,Filipino,Chinese,Vietnamese": 0.3181818181818182, "English,Filipino,Malay,Vietnamese": 0.32386363636363635, "English,Spanish,Chinese,Malay": 0.38636363636363635, "English,Spanish,Chinese,Vietnamese": 0.375, "English,Spanish,Malay,Vietnamese": 0.3977272727272727, "English,Chinese,Malay,Vietnamese": 0.3409090909090909, "Filipino,Spanish,Chinese,Malay": 0.32386363636363635, "Filipino,Spanish,Chinese,Vietnamese": 0.32386363636363635, "Filipino,Spanish,Malay,Vietnamese": 0.3409090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.32954545454545453, "Spanish,Chinese,Malay,Vietnamese": 0.3465909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2727272727272727, "Indonesian,English,Filipino,Spanish,Malay": 0.3125, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2840909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.2840909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Filipino,Malay,Vietnamese": 0.2897727272727273, "Indonesian,English,Spanish,Chinese,Malay": 0.32386363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3068181818181818, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3465909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.26136363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish,Chinese,Malay": 0.29545454545454547, "English,Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.2840909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.24431818181818182, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.25, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091 } }, "AC3_2": 0.5349458955274844, "AC3_3": 0.4603894961983203, "AC3_4": 0.4066322027000697, "AC3_5": 0.3650482896280321, "AC3_6": 0.33181104605183825, "AC3_7": 0.3045923148586084 }, "prompt_3": { "overall_acc": 0.4715909090909091, "language_acc": { "Indonesian": 0.4431818181818182, "English": 0.5681818181818182, "Filipino": 0.39204545454545453, "Spanish": 0.5340909090909091, "Chinese": 0.4772727272727273, "Malay": 0.4659090909090909, "Vietnamese": 0.42045454545454547 }, "consistency_score_2": 0.554112554112554, "consistency_score_3": 0.38165584415584414, "consistency_score_4": 0.2931818181818182, "consistency_score_5": 0.23971861471861478, "consistency_score_6": 0.20454545454545456, "consistency_score_7": 0.18181818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5227272727272727, "Indonesian,Filipino": 0.5568181818181818, "Indonesian,Spanish": 0.5965909090909091, "Indonesian,Chinese": 0.5113636363636364, "Indonesian,Malay": 0.6704545454545454, "Indonesian,Vietnamese": 0.5397727272727273, "English,Filipino": 0.4431818181818182, "English,Spanish": 0.6818181818181818, "English,Chinese": 0.5284090909090909, "English,Malay": 0.5340909090909091, "English,Vietnamese": 0.5681818181818182, "Filipino,Spanish": 0.5340909090909091, "Filipino,Chinese": 0.4602272727272727, "Filipino,Malay": 0.6193181818181818, "Filipino,Vietnamese": 0.5227272727272727, "Spanish,Chinese": 0.48295454545454547, "Spanish,Malay": 0.6306818181818182, "Spanish,Vietnamese": 0.6022727272727273, "Chinese,Malay": 0.5, "Chinese,Vietnamese": 0.5625, "Malay,Vietnamese": 0.5681818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.3352272727272727, "Indonesian,English,Spanish": 0.4318181818181818, "Indonesian,English,Chinese": 0.3352272727272727, "Indonesian,English,Malay": 0.4090909090909091, "Indonesian,English,Vietnamese": 0.36363636363636365, "Indonesian,Filipino,Spanish": 0.4034090909090909, "Indonesian,Filipino,Chinese": 0.3125, "Indonesian,Filipino,Malay": 0.4772727272727273, "Indonesian,Filipino,Vietnamese": 0.36363636363636365, "Indonesian,Spanish,Chinese": 0.3465909090909091, "Indonesian,Spanish,Malay": 0.48863636363636365, "Indonesian,Spanish,Vietnamese": 0.4147727272727273, "Indonesian,Chinese,Malay": 0.3806818181818182, "Indonesian,Chinese,Vietnamese": 0.36363636363636365, "Indonesian,Malay,Vietnamese": 0.4431818181818182, "English,Filipino,Spanish": 0.3806818181818182, "English,Filipino,Chinese": 0.2897727272727273, "English,Filipino,Malay": 0.35795454545454547, "English,Filipino,Vietnamese": 0.32386363636363635, "English,Spanish,Chinese": 0.36363636363636365, "English,Spanish,Malay": 0.4602272727272727, "English,Spanish,Vietnamese": 0.4431818181818182, "English,Chinese,Malay": 0.3352272727272727, "English,Chinese,Vietnamese": 0.3693181818181818, "English,Malay,Vietnamese": 0.3693181818181818, "Filipino,Spanish,Chinese": 0.3125, "Filipino,Spanish,Malay": 0.42045454545454547, "Filipino,Spanish,Vietnamese": 0.38636363636363635, "Filipino,Chinese,Malay": 0.3693181818181818, "Filipino,Chinese,Vietnamese": 0.3465909090909091, "Filipino,Malay,Vietnamese": 0.4090909090909091, "Spanish,Chinese,Malay": 0.3693181818181818, "Spanish,Chinese,Vietnamese": 0.36363636363636365, "Spanish,Malay,Vietnamese": 0.4431818181818182, "Chinese,Malay,Vietnamese": 0.375 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.29545454545454547, "Indonesian,English,Filipino,Chinese": 0.23863636363636365, "Indonesian,English,Filipino,Malay": 0.3181818181818182, "Indonesian,English,Filipino,Vietnamese": 0.26704545454545453, "Indonesian,English,Spanish,Chinese": 0.2784090909090909, "Indonesian,English,Spanish,Malay": 0.36363636363636365, "Indonesian,English,Spanish,Vietnamese": 0.32386363636363635, "Indonesian,English,Chinese,Malay": 0.2784090909090909, "Indonesian,English,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Malay,Vietnamese": 0.3068181818181818, "Indonesian,Filipino,Spanish,Chinese": 0.23863636363636365, "Indonesian,Filipino,Spanish,Malay": 0.36363636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Chinese,Malay": 0.30113636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.26136363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.3409090909090909, "Indonesian,Spanish,Chinese,Malay": 0.30113636363636365, "Indonesian,Spanish,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.36363636363636365, "Indonesian,Chinese,Malay,Vietnamese": 0.3068181818181818, "English,Filipino,Spanish,Chinese": 0.24431818181818182, "English,Filipino,Spanish,Malay": 0.3125, "English,Filipino,Spanish,Vietnamese": 0.2840909090909091, "English,Filipino,Chinese,Malay": 0.2556818181818182, "English,Filipino,Chinese,Vietnamese": 0.25, "English,Filipino,Malay,Vietnamese": 0.2784090909090909, "English,Spanish,Chinese,Malay": 0.2840909090909091, "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, "English,Spanish,Malay,Vietnamese": 0.32386363636363635, "English,Chinese,Malay,Vietnamese": 0.2727272727272727, "Filipino,Spanish,Chinese,Malay": 0.2727272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.3068181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.2784090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.24431818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 } }, "AC3_2": 0.5095321454624216, "AC3_3": 0.421883648075642, "AC3_4": 0.36157638790355184, "AC3_5": 0.3178619592208479, "AC3_6": 0.2853323147018791, "AC3_7": 0.26245059284520916 }, "prompt_4": { "overall_acc": 0.4813311688311689, "language_acc": { "Indonesian": 0.4431818181818182, "English": 0.5681818181818182, "Filipino": 0.4090909090909091, "Spanish": 0.5397727272727273, "Chinese": 0.5056818181818182, "Malay": 0.4318181818181818, "Vietnamese": 0.4715909090909091 }, "consistency_score_2": 0.5827922077922079, "consistency_score_3": 0.4183441558441558, "consistency_score_4": 0.32987012987012987, "consistency_score_5": 0.2740800865800866, "consistency_score_6": 0.2362012987012987, "consistency_score_7": 0.21022727272727273, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5511363636363636, "Indonesian,Filipino": 0.5397727272727273, "Indonesian,Spanish": 0.6477272727272727, "Indonesian,Chinese": 0.5340909090909091, "Indonesian,Malay": 0.6761363636363636, "Indonesian,Vietnamese": 0.5852272727272727, "English,Filipino": 0.48863636363636365, "English,Spanish": 0.7556818181818182, "English,Chinese": 0.5965909090909091, "English,Malay": 0.5738636363636364, "English,Vietnamese": 0.6363636363636364, "Filipino,Spanish": 0.5284090909090909, "Filipino,Chinese": 0.4772727272727273, "Filipino,Malay": 0.6136363636363636, "Filipino,Vietnamese": 0.5227272727272727, "Spanish,Chinese": 0.5738636363636364, "Spanish,Malay": 0.6534090909090909, "Spanish,Vietnamese": 0.625, "Chinese,Malay": 0.5, "Chinese,Vietnamese": 0.6079545454545454, "Malay,Vietnamese": 0.5511363636363636 }, "3_combine": { "Indonesian,English,Filipino": 0.36363636363636365, "Indonesian,English,Spanish": 0.5113636363636364, "Indonesian,English,Chinese": 0.38636363636363635, "Indonesian,English,Malay": 0.44886363636363635, "Indonesian,English,Vietnamese": 0.4375, "Indonesian,Filipino,Spanish": 0.4090909090909091, "Indonesian,Filipino,Chinese": 0.32386363636363635, "Indonesian,Filipino,Malay": 0.45454545454545453, "Indonesian,Filipino,Vietnamese": 0.375, "Indonesian,Spanish,Chinese": 0.42045454545454547, "Indonesian,Spanish,Malay": 0.5284090909090909, "Indonesian,Spanish,Vietnamese": 0.4659090909090909, "Indonesian,Chinese,Malay": 0.39204545454545453, "Indonesian,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,Malay,Vietnamese": 0.4375, "English,Filipino,Spanish": 0.4318181818181818, "English,Filipino,Chinese": 0.3465909090909091, "English,Filipino,Malay": 0.38636363636363635, "English,Filipino,Vietnamese": 0.3806818181818182, "English,Spanish,Chinese": 0.4943181818181818, "English,Spanish,Malay": 0.5227272727272727, "English,Spanish,Vietnamese": 0.5340909090909091, "English,Chinese,Malay": 0.375, "English,Chinese,Vietnamese": 0.45454545454545453, "English,Malay,Vietnamese": 0.42613636363636365, "Filipino,Spanish,Chinese": 0.3465909090909091, "Filipino,Spanish,Malay": 0.4318181818181818, "Filipino,Spanish,Vietnamese": 0.3806818181818182, "Filipino,Chinese,Malay": 0.3465909090909091, "Filipino,Chinese,Vietnamese": 0.3522727272727273, "Filipino,Malay,Vietnamese": 0.39204545454545453, "Spanish,Chinese,Malay": 0.4090909090909091, "Spanish,Chinese,Vietnamese": 0.4375, "Spanish,Malay,Vietnamese": 0.45454545454545453, "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.3465909090909091, "Indonesian,English,Filipino,Chinese": 0.26704545454545453, "Indonesian,English,Filipino,Malay": 0.32386363636363635, "Indonesian,English,Filipino,Vietnamese": 0.30113636363636365, "Indonesian,English,Spanish,Chinese": 0.36363636363636365, "Indonesian,English,Spanish,Malay": 0.42613636363636365, "Indonesian,English,Spanish,Vietnamese": 0.4147727272727273, "Indonesian,English,Chinese,Malay": 0.3125, "Indonesian,English,Chinese,Vietnamese": 0.32954545454545453, "Indonesian,English,Malay,Vietnamese": 0.35795454545454547, "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Malay": 0.375, "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, "Indonesian,Filipino,Chinese,Malay": 0.2784090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Spanish,Chinese,Malay": 0.3465909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.3522727272727273, "Indonesian,Spanish,Malay,Vietnamese": 0.38636363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.3181818181818182, "English,Filipino,Spanish,Chinese": 0.3125, "English,Filipino,Spanish,Malay": 0.35795454545454547, "English,Filipino,Spanish,Vietnamese": 0.3352272727272727, "English,Filipino,Chinese,Malay": 0.2784090909090909, "English,Filipino,Chinese,Vietnamese": 0.2840909090909091, "English,Filipino,Malay,Vietnamese": 0.3125, "English,Spanish,Chinese,Malay": 0.35795454545454547, "English,Spanish,Chinese,Vietnamese": 0.3977272727272727, "English,Spanish,Malay,Vietnamese": 0.39204545454545453, "English,Chinese,Malay,Vietnamese": 0.3181818181818182, "Filipino,Spanish,Chinese,Malay": 0.2840909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.2840909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.2897727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2556818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.3125, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2840909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.23863636363636365, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,English,Filipino,Malay,Vietnamese": 0.26704545454545453, "Indonesian,English,Spanish,Chinese,Malay": 0.3068181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3181818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3409090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.25, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish,Chinese,Malay": 0.26136363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.26136363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.25 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273 } }, "AC3_2": 0.5272246822094617, "AC3_3": 0.447632775871545, "AC3_4": 0.39146085048803886, "AC3_5": 0.34927541114715466, "AC3_6": 0.31689450251210277, "AC3_7": 0.29264031152405184 }, "prompt_5": { "overall_acc": 0.45454545454545453, "language_acc": { "Indonesian": 0.4375, "English": 0.5056818181818182, "Filipino": 0.4147727272727273, "Spanish": 0.48863636363636365, "Chinese": 0.42613636363636365, "Malay": 0.44886363636363635, "Vietnamese": 0.4602272727272727 }, "consistency_score_2": 0.5454545454545455, "consistency_score_3": 0.3748376623376623, "consistency_score_4": 0.2870129870129871, "consistency_score_5": 0.23376623376623382, "consistency_score_6": 0.19724025974025977, "consistency_score_7": 0.17045454545454544, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5113636363636364, "Indonesian,Filipino": 0.5625, "Indonesian,Spanish": 0.5852272727272727, "Indonesian,Chinese": 0.5113636363636364, "Indonesian,Malay": 0.6420454545454546, "Indonesian,Vietnamese": 0.5681818181818182, "English,Filipino": 0.4375, "English,Spanish": 0.6534090909090909, "English,Chinese": 0.5, "English,Malay": 0.5113636363636364, "English,Vietnamese": 0.5170454545454546, "Filipino,Spanish": 0.5113636363636364, "Filipino,Chinese": 0.4602272727272727, "Filipino,Malay": 0.6079545454545454, "Filipino,Vietnamese": 0.5227272727272727, "Spanish,Chinese": 0.5227272727272727, "Spanish,Malay": 0.6136363636363636, "Spanish,Vietnamese": 0.6022727272727273, "Chinese,Malay": 0.5284090909090909, "Chinese,Vietnamese": 0.5, "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.3352272727272727, "Indonesian,English,Spanish": 0.4090909090909091, "Indonesian,English,Chinese": 0.32386363636363635, "Indonesian,English,Malay": 0.39204545454545453, "Indonesian,English,Vietnamese": 0.3522727272727273, "Indonesian,Filipino,Spanish": 0.3977272727272727, "Indonesian,Filipino,Chinese": 0.32386363636363635, "Indonesian,Filipino,Malay": 0.4602272727272727, "Indonesian,Filipino,Vietnamese": 0.38636363636363635, "Indonesian,Spanish,Chinese": 0.35795454545454547, "Indonesian,Spanish,Malay": 0.4772727272727273, "Indonesian,Spanish,Vietnamese": 0.42045454545454547, "Indonesian,Chinese,Malay": 0.3977272727272727, "Indonesian,Chinese,Vietnamese": 0.3465909090909091, "Indonesian,Malay,Vietnamese": 0.4602272727272727, "English,Filipino,Spanish": 0.35795454545454547, "English,Filipino,Chinese": 0.29545454545454547, "English,Filipino,Malay": 0.32954545454545453, "English,Filipino,Vietnamese": 0.3125, "English,Spanish,Chinese": 0.375, "English,Spanish,Malay": 0.4318181818181818, "English,Spanish,Vietnamese": 0.42613636363636365, "English,Chinese,Malay": 0.32386363636363635, "English,Chinese,Vietnamese": 0.3068181818181818, "English,Malay,Vietnamese": 0.36363636363636365, "Filipino,Spanish,Chinese": 0.32954545454545453, "Filipino,Spanish,Malay": 0.42045454545454547, "Filipino,Spanish,Vietnamese": 0.375, "Filipino,Chinese,Malay": 0.3522727272727273, "Filipino,Chinese,Vietnamese": 0.3181818181818182, "Filipino,Malay,Vietnamese": 0.4090909090909091, "Spanish,Chinese,Malay": 0.3806818181818182, "Spanish,Chinese,Vietnamese": 0.36363636363636365, "Spanish,Malay,Vietnamese": 0.4431818181818182, "Chinese,Malay,Vietnamese": 0.36363636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2897727272727273, "Indonesian,English,Filipino,Chinese": 0.23863636363636365, "Indonesian,English,Filipino,Malay": 0.30113636363636365, "Indonesian,English,Filipino,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Chinese": 0.2727272727272727, "Indonesian,English,Spanish,Malay": 0.3522727272727273, "Indonesian,English,Spanish,Vietnamese": 0.3181818181818182, "Indonesian,English,Chinese,Malay": 0.2727272727272727, "Indonesian,English,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,English,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.25, "Indonesian,Filipino,Spanish,Malay": 0.36363636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, "Indonesian,Filipino,Chinese,Malay": 0.2897727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.25, "Indonesian,Filipino,Malay,Vietnamese": 0.35795454545454547, "Indonesian,Spanish,Chinese,Malay": 0.30113636363636365, "Indonesian,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.3068181818181818, "English,Filipino,Spanish,Chinese": 0.24431818181818182, "English,Filipino,Spanish,Malay": 0.29545454545454547, "English,Filipino,Spanish,Vietnamese": 0.2727272727272727, "English,Filipino,Chinese,Malay": 0.22727272727272727, "English,Filipino,Chinese,Vietnamese": 0.2215909090909091, "English,Filipino,Malay,Vietnamese": 0.2727272727272727, "English,Spanish,Chinese,Malay": 0.2897727272727273, "English,Spanish,Chinese,Vietnamese": 0.26136363636363635, "English,Spanish,Malay,Vietnamese": 0.32954545454545453, "English,Chinese,Malay,Vietnamese": 0.2556818181818182, "Filipino,Spanish,Chinese,Malay": 0.2727272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.3409090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.2727272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.2727272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.2159090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, "Indonesian,English,Filipino,Malay,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.24431818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.29545454545454547, "Indonesian,English,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.3068181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 } }, "AC3_2": 0.4958677685454546, "AC3_3": 0.41086140317827724, "AC3_4": 0.35185480014360443, "AC3_5": 0.30874785587281145, "AC3_6": 0.2751047208906576, "AC3_7": 0.2479338842578512 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6407766990291263 }, "prompt_2": { "accuracy": 0.6213592233009708 }, "prompt_3": { "accuracy": 0.6019417475728155 }, "prompt_4": { "accuracy": 0.6019417475728155 }, "prompt_5": { "accuracy": 0.6407766990291263 } }, "cn_eval": { "prompt_1": { "accuracy": 0.41904761904761906 }, "prompt_2": { "accuracy": 0.4857142857142857 }, "prompt_3": { "accuracy": 0.5238095238095238 }, "prompt_4": { "accuracy": 0.4666666666666667 }, "prompt_5": { "accuracy": 0.4857142857142857 } }, "us_eval": { "prompt_1": { "accuracy": 0.7009345794392523 }, "prompt_2": { "accuracy": 0.7009345794392523 }, "prompt_3": { "accuracy": 0.6915887850467289 }, "prompt_4": { "accuracy": 0.7009345794392523 }, "prompt_5": { "accuracy": 0.7009345794392523 } }, "ph_eval": { "prompt_1": { "accuracy": 0.52, "category_acc": { "brand": 0.5, "demographics": 0.6, "biology": 0.5, "history": 0.4, "literature": 0.2, "politics": 0.7, "culture": 0.8, "film": 0.5, "law": 0.5, "geography": 0.6 } }, "prompt_2": { "accuracy": 0.52, "category_acc": { "brand": 0.6, "demographics": 0.4, "biology": 0.6, "history": 0.4, "literature": 0.2, "politics": 0.6, "culture": 0.7, "film": 0.6, "law": 0.6, "geography": 0.5 } }, "prompt_3": { "accuracy": 0.52, "category_acc": { "brand": 0.5, "demographics": 0.2, "biology": 0.5, "history": 0.5333333333333333, "literature": 0.3, "politics": 0.7, "culture": 0.8, "film": 0.4, "law": 0.6, "geography": 0.5 } }, "prompt_4": { "accuracy": 0.58, "category_acc": { "brand": 0.6, "demographics": 0.4, "biology": 0.6, "history": 0.4666666666666667, "literature": 0.3, "politics": 0.7, "culture": 0.8, "film": 0.5, "law": 0.7, "geography": 0.7 } }, "prompt_5": { "accuracy": 0.59, "category_acc": { "brand": 0.5, "demographics": 0.4, "biology": 0.6, "history": 0.6, "literature": 0.2, "politics": 0.9, "culture": 0.8, "film": 0.5, "law": 0.7, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.06094069384356101 }, "prompt_2": { "bleu_score": 0.06207526753612465 }, "prompt_3": { "bleu_score": 0.06192642106229149 }, "prompt_4": { "bleu_score": 0.060092982025264 }, "prompt_5": { "bleu_score": 0.05469202971301356 } }, "indommlu": { "prompt_1": { "accuracy": 0.5280726350223647, "category_acc": { "History": 0.5240963855421686, "Geography": 0.5020408163265306, "Lampungic": 0.3741496598639456, "Social science": 0.7495826377295493, "Balinese": 0.30148619957537154, "Makassarese": 0.3763440860215054, "Banjarese": 0.4444444444444444, "Chemistry": 0.31386861313868614, "Biology": 0.5029585798816568, "Science": 0.6780185758513931, "Christian religion": 0.6616915422885572, "Art": 0.5940099833610649, "Islam religion": 0.65149359886202, "Hindu religion": 0.5466666666666666, "Madurese": 0.288135593220339, "Sport": 0.5337837837837838, "Indonesian language": 0.587173100871731, "Physics": 0.4686868686868687, "Minangkabau culture": 0.457286432160804, "Dayak language": 0.3486238532110092, "Sociology": 0.5463709677419355, "Economy": 0.5020491803278688, "Sundanese": 0.4606741573033708, "Javanese": 0.4092741935483871, "Civic education": 0.6151645207439199 } }, "prompt_2": { "accuracy": 0.530676280125509, "category_acc": { "History": 0.5160642570281124, "Geography": 0.5, "Lampungic": 0.35374149659863946, "Social science": 0.7612687813021702, "Balinese": 0.28662420382165604, "Makassarese": 0.3655913978494624, "Banjarese": 0.4375, "Chemistry": 0.345985401459854, "Biology": 0.5076923076923077, "Science": 0.6904024767801857, "Christian religion": 0.6567164179104478, "Art": 0.6073211314475874, "Islam religion": 0.6486486486486487, "Hindu religion": 0.58, "Madurese": 0.3593220338983051, "Sport": 0.5337837837837838, "Indonesian language": 0.5840597758405978, "Physics": 0.4909090909090909, "Minangkabau culture": 0.4221105527638191, "Dayak language": 0.24770642201834864, "Sociology": 0.5504032258064516, "Economy": 0.4979508196721312, "Sundanese": 0.4520311149524633, "Javanese": 0.42338709677419356, "Civic education": 0.6065808297567954 } }, "prompt_3": { "accuracy": 0.5258695507043194, "category_acc": { "History": 0.5140562248995983, "Geography": 0.5142857142857142, "Lampungic": 0.3469387755102041, "Social science": 0.7462437395659433, "Balinese": 0.29723991507430997, "Makassarese": 0.3602150537634409, "Banjarese": 0.4097222222222222, "Chemistry": 0.31386861313868614, "Biology": 0.5183431952662721, "Science": 0.6790505675954592, "Christian religion": 0.6517412935323383, "Art": 0.6073211314475874, "Islam religion": 0.635846372688478, "Hindu religion": 0.58, "Madurese": 0.3423728813559322, "Sport": 0.527027027027027, "Indonesian language": 0.5874844333748444, "Physics": 0.4585858585858586, "Minangkabau culture": 0.46733668341708545, "Dayak language": 0.26605504587155965, "Sociology": 0.5362903225806451, "Economy": 0.48565573770491804, "Sundanese": 0.44511668107173724, "Javanese": 0.41330645161290325, "Civic education": 0.6022889842632332 } }, "prompt_4": { "accuracy": 0.5315441618265572, "category_acc": { "History": 0.5261044176706827, "Geography": 0.5244897959183673, "Lampungic": 0.36054421768707484, "Social science": 0.7479131886477463, "Balinese": 0.3205944798301486, "Makassarese": 0.4032258064516129, "Banjarese": 0.4166666666666667, "Chemistry": 0.3284671532846715, "Biology": 0.5360946745562131, "Science": 0.6862745098039216, "Christian religion": 0.6766169154228856, "Art": 0.6023294509151415, "Islam religion": 0.6273115220483642, "Hindu religion": 0.56, "Madurese": 0.3389830508474576, "Sport": 0.527027027027027, "Indonesian language": 0.5896637608966376, "Physics": 0.45252525252525255, "Minangkabau culture": 0.4623115577889447, "Dayak language": 0.23853211009174313, "Sociology": 0.5725806451612904, "Economy": 0.5122950819672131, "Sundanese": 0.452895419187554, "Javanese": 0.39818548387096775, "Civic education": 0.6051502145922747 } }, "prompt_5": { "accuracy": 0.5029040656919688, "category_acc": { "History": 0.4779116465863454, "Geography": 0.4714285714285714, "Lampungic": 0.38095238095238093, "Social science": 0.6944908180300501, "Balinese": 0.3057324840764331, "Makassarese": 0.2903225806451613, "Banjarese": 0.4027777777777778, "Chemistry": 0.2934306569343066, "Biology": 0.46153846153846156, "Science": 0.6295149638802889, "Christian religion": 0.6567164179104478, "Art": 0.589018302828619, "Islam religion": 0.6443812233285917, "Hindu religion": 0.52, "Madurese": 0.30847457627118646, "Sport": 0.5135135135135135, "Indonesian language": 0.5716064757160647, "Physics": 0.4484848484848485, "Minangkabau culture": 0.4271356783919598, "Dayak language": 0.29357798165137616, "Sociology": 0.48185483870967744, "Economy": 0.4672131147540984, "Sundanese": 0.4399308556611927, "Javanese": 0.39818548387096775, "Civic education": 0.5793991416309013 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.07998569558291352 }, "prompt_2": { "bleu_score": 0.08178489772334997 }, "prompt_3": { "bleu_score": 0.08148612648063638 }, "prompt_4": { "bleu_score": 0.07978071248841677 }, "prompt_5": { "bleu_score": 0.08665723913254166 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.06595796158495684 }, "prompt_2": { "bleu_score": 0.0659317494666598 }, "prompt_3": { "bleu_score": 0.06631225923987717 }, "prompt_4": { "bleu_score": 0.06562445607881094 }, "prompt_5": { "bleu_score": 0.07261211813311091 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.0502826099860938 }, "prompt_2": { "bleu_score": 0.05134705353375384 }, "prompt_3": { "bleu_score": 0.051760820535713056 }, "prompt_4": { "bleu_score": 0.05136651836443981 }, "prompt_5": { "bleu_score": 0.05595680934355571 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.08113017898863324 }, "prompt_2": { "bleu_score": 0.08240154342677156 }, "prompt_3": { "bleu_score": 0.08322691501291536 }, "prompt_4": { "bleu_score": 0.07956020114017891 }, "prompt_5": { "bleu_score": 0.08635159030424207 } }, "mmlu": { "prompt_1": { "accuracy": 0.6254375729288215 }, "prompt_2": { "accuracy": 0.646441073512252 }, "prompt_3": { "accuracy": 0.6277712952158693 }, "prompt_4": { "accuracy": 0.6394399066511085 }, "prompt_5": { "accuracy": 0.588098016336056 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.619449410082231, "category_acc": { "high_school_european_history": 0.7378048780487805, "business_ethics": 0.5656565656565656, "clinical_knowledge": 0.6856060606060606, "medical_genetics": 0.7171717171717171, "high_school_us_history": 0.7635467980295566, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.7796610169491526, "virology": 0.4909090909090909, "high_school_microeconomics": 0.6835443037974683, "econometrics": 0.4336283185840708, "college_computer_science": 0.4444444444444444, "high_school_biology": 0.7216828478964401, "abstract_algebra": 0.32323232323232326, "professional_accounting": 0.47330960854092524, "philosophy": 0.6483870967741936, "professional_medicine": 0.7343173431734318, "nutrition": 0.7475409836065574, "global_facts": 0.5252525252525253, "machine_learning": 0.3783783783783784, "security_studies": 0.6352459016393442, "public_relations": 0.6146788990825688, "professional_psychology": 0.6415711947626841, "prehistory": 0.7058823529411765, "anatomy": 0.664179104477612, "human_sexuality": 0.7230769230769231, "college_medicine": 0.6046511627906976, "high_school_government_and_politics": 0.8333333333333334, "college_chemistry": 0.41414141414141414, "logical_fallacies": 0.7469135802469136, "high_school_geography": 0.7360406091370558, "elementary_mathematics": 0.7029177718832891, "human_aging": 0.6621621621621622, "college_mathematics": 0.23232323232323232, "high_school_psychology": 0.8051470588235294, "formal_logic": 0.456, "high_school_statistics": 0.4511627906976744, "international_law": 0.75, "high_school_mathematics": 0.3048327137546468, "high_school_computer_science": 0.696969696969697, "conceptual_physics": 0.5555555555555556, "miscellaneous": 0.8414322250639387, "high_school_chemistry": 0.4801980198019802, "marketing": 0.8497854077253219, "professional_law": 0.4592302674494455, "management": 0.7647058823529411, "college_physics": 0.3564356435643564, "jurisprudence": 0.7383177570093458, "world_religions": 0.8, "sociology": 0.8, "us_foreign_policy": 0.8585858585858586, "high_school_macroeconomics": 0.5989717223650386, "computer_security": 0.696969696969697, "moral_scenarios": 0.4004474272930649, "moral_disputes": 0.6608695652173913, "electrical_engineering": 0.625, "astronomy": 0.6887417218543046, "college_biology": 0.6923076923076923 } }, "prompt_2": { "accuracy": 0.6249553092599214, "category_acc": { "high_school_european_history": 0.7439024390243902, "business_ethics": 0.6464646464646465, "clinical_knowledge": 0.7424242424242424, "medical_genetics": 0.7676767676767676, "high_school_us_history": 0.7783251231527094, "high_school_physics": 0.4066666666666667, "high_school_world_history": 0.8220338983050848, "virology": 0.49696969696969695, "high_school_microeconomics": 0.7088607594936709, "econometrics": 0.45132743362831856, "college_computer_science": 0.47474747474747475, "high_school_biology": 0.7734627831715211, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.5088967971530249, "philosophy": 0.6516129032258065, "professional_medicine": 0.7527675276752768, "nutrition": 0.7344262295081967, "global_facts": 0.45454545454545453, "machine_learning": 0.4864864864864865, "security_studies": 0.6721311475409836, "public_relations": 0.6605504587155964, "professional_psychology": 0.6612111292962357, "prehistory": 0.7120743034055728, "anatomy": 0.6417910447761194, "human_sexuality": 0.7230769230769231, "college_medicine": 0.6162790697674418, "high_school_government_and_politics": 0.859375, "college_chemistry": 0.37373737373737376, "logical_fallacies": 0.7407407407407407, "high_school_geography": 0.7766497461928934, "elementary_mathematics": 0.7082228116710876, "human_aging": 0.6666666666666666, "college_mathematics": 0.3333333333333333, "high_school_psychology": 0.8198529411764706, "formal_logic": 0.504, "high_school_statistics": 0.5209302325581395, "international_law": 0.7583333333333333, "high_school_mathematics": 0.35687732342007433, "high_school_computer_science": 0.7070707070707071, "conceptual_physics": 0.5769230769230769, "miscellaneous": 0.8286445012787724, "high_school_chemistry": 0.5148514851485149, "marketing": 0.8755364806866953, "professional_law": 0.4611872146118721, "management": 0.8137254901960784, "college_physics": 0.37623762376237624, "jurisprudence": 0.7383177570093458, "world_religions": 0.788235294117647, "sociology": 0.795, "us_foreign_policy": 0.8181818181818182, "high_school_macroeconomics": 0.5938303341902313, "computer_security": 0.6868686868686869, "moral_scenarios": 0.25615212527964204, "moral_disputes": 0.6840579710144927, "electrical_engineering": 0.5625, "astronomy": 0.7152317880794702, "college_biology": 0.7202797202797203 } }, "prompt_3": { "accuracy": 0.6251698248122989, "category_acc": { "high_school_european_history": 0.7317073170731707, "business_ethics": 0.696969696969697, "clinical_knowledge": 0.7234848484848485, "medical_genetics": 0.7474747474747475, "high_school_us_history": 0.7684729064039408, "high_school_physics": 0.36666666666666664, "high_school_world_history": 0.809322033898305, "virology": 0.503030303030303, "high_school_microeconomics": 0.7215189873417721, "econometrics": 0.46017699115044247, "college_computer_science": 0.43434343434343436, "high_school_biology": 0.7443365695792881, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.5088967971530249, "philosophy": 0.6645161290322581, "professional_medicine": 0.7490774907749077, "nutrition": 0.7180327868852459, "global_facts": 0.48484848484848486, "machine_learning": 0.4864864864864865, "security_studies": 0.6721311475409836, "public_relations": 0.6330275229357798, "professional_psychology": 0.6350245499181669, "prehistory": 0.7151702786377709, "anatomy": 0.6567164179104478, "human_sexuality": 0.7461538461538462, "college_medicine": 0.5988372093023255, "high_school_government_and_politics": 0.859375, "college_chemistry": 0.47474747474747475, "logical_fallacies": 0.7592592592592593, "high_school_geography": 0.7461928934010152, "elementary_mathematics": 0.7294429708222812, "human_aging": 0.6891891891891891, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.8051470588235294, "formal_logic": 0.536, "high_school_statistics": 0.5023255813953489, "international_law": 0.7416666666666667, "high_school_mathematics": 0.3345724907063197, "high_school_computer_science": 0.6565656565656566, "conceptual_physics": 0.6068376068376068, "miscellaneous": 0.8350383631713555, "high_school_chemistry": 0.5198019801980198, "marketing": 0.8540772532188842, "professional_law": 0.4683626875407697, "management": 0.7647058823529411, "college_physics": 0.37623762376237624, "jurisprudence": 0.719626168224299, "world_religions": 0.8058823529411765, "sociology": 0.8, "us_foreign_policy": 0.8686868686868687, "high_school_macroeconomics": 0.583547557840617, "computer_security": 0.6767676767676768, "moral_scenarios": 0.2897091722595078, "moral_disputes": 0.6753623188405797, "electrical_engineering": 0.5763888888888888, "astronomy": 0.695364238410596, "college_biology": 0.7272727272727273 } }, "prompt_4": { "accuracy": 0.6171612441902038, "category_acc": { "high_school_european_history": 0.7317073170731707, "business_ethics": 0.6565656565656566, "clinical_knowledge": 0.6590909090909091, "medical_genetics": 0.7575757575757576, "high_school_us_history": 0.7931034482758621, "high_school_physics": 0.34, "high_school_world_history": 0.8008474576271186, "virology": 0.49696969696969695, "high_school_microeconomics": 0.7088607594936709, "econometrics": 0.3893805309734513, "college_computer_science": 0.3939393939393939, "high_school_biology": 0.7249190938511327, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.5480427046263345, "philosophy": 0.6774193548387096, "professional_medicine": 0.6678966789667896, "nutrition": 0.7114754098360656, "global_facts": 0.5050505050505051, "machine_learning": 0.35135135135135137, "security_studies": 0.6721311475409836, "public_relations": 0.6513761467889908, "professional_psychology": 0.630114566284779, "prehistory": 0.7120743034055728, "anatomy": 0.6119402985074627, "human_sexuality": 0.6923076923076923, "college_medicine": 0.6046511627906976, "high_school_government_and_politics": 0.8229166666666666, "college_chemistry": 0.3939393939393939, "logical_fallacies": 0.7654320987654321, "high_school_geography": 0.7309644670050761, "elementary_mathematics": 0.6816976127320955, "human_aging": 0.6486486486486487, "college_mathematics": 0.35353535353535354, "high_school_psychology": 0.7886029411764706, "formal_logic": 0.488, "high_school_statistics": 0.4744186046511628, "international_law": 0.775, "high_school_mathematics": 0.38661710037174724, "high_school_computer_science": 0.6767676767676768, "conceptual_physics": 0.594017094017094, "miscellaneous": 0.7877237851662404, "high_school_chemistry": 0.5, "marketing": 0.8283261802575107, "professional_law": 0.47619047619047616, "management": 0.803921568627451, "college_physics": 0.4158415841584158, "jurisprudence": 0.7476635514018691, "world_religions": 0.8, "sociology": 0.8, "us_foreign_policy": 0.8383838383838383, "high_school_macroeconomics": 0.5732647814910026, "computer_security": 0.6161616161616161, "moral_scenarios": 0.3434004474272931, "moral_disputes": 0.6695652173913044, "electrical_engineering": 0.6180555555555556, "astronomy": 0.7218543046357616, "college_biology": 0.6503496503496503 } }, "prompt_5": { "accuracy": 0.5823382195209152, "category_acc": { "high_school_european_history": 0.7134146341463414, "business_ethics": 0.6161616161616161, "clinical_knowledge": 0.6401515151515151, "medical_genetics": 0.7373737373737373, "high_school_us_history": 0.7783251231527094, "high_school_physics": 0.32666666666666666, "high_school_world_history": 0.7923728813559322, "virology": 0.4909090909090909, "high_school_microeconomics": 0.6118143459915611, "econometrics": 0.35398230088495575, "college_computer_science": 0.36363636363636365, "high_school_biology": 0.6472491909385113, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.4483985765124555, "philosophy": 0.6580645161290323, "professional_medicine": 0.6457564575645757, "nutrition": 0.6786885245901639, "global_facts": 0.494949494949495, "machine_learning": 0.36936936936936937, "security_studies": 0.5983606557377049, "public_relations": 0.5504587155963303, "professional_psychology": 0.6104746317512275, "prehistory": 0.6687306501547987, "anatomy": 0.5895522388059702, "human_sexuality": 0.5923076923076923, "college_medicine": 0.5348837209302325, "high_school_government_and_politics": 0.796875, "college_chemistry": 0.31313131313131315, "logical_fallacies": 0.7345679012345679, "high_school_geography": 0.6700507614213198, "elementary_mathematics": 0.726790450928382, "human_aging": 0.6396396396396397, "college_mathematics": 0.21212121212121213, "high_school_psychology": 0.7408088235294118, "formal_logic": 0.4, "high_school_statistics": 0.40930232558139534, "international_law": 0.7083333333333334, "high_school_mathematics": 0.2936802973977695, "high_school_computer_science": 0.6666666666666666, "conceptual_physics": 0.5470085470085471, "miscellaneous": 0.7902813299232737, "high_school_chemistry": 0.41089108910891087, "marketing": 0.8111587982832618, "professional_law": 0.44879321591650356, "management": 0.7549019607843137, "college_physics": 0.3564356435643564, "jurisprudence": 0.6915887850467289, "world_religions": 0.8176470588235294, "sociology": 0.775, "us_foreign_policy": 0.8282828282828283, "high_school_macroeconomics": 0.5475578406169666, "computer_security": 0.6666666666666666, "moral_scenarios": 0.30648769574944074, "moral_disputes": 0.6347826086956522, "electrical_engineering": 0.6111111111111112, "astronomy": 0.6357615894039735, "college_biology": 0.6013986013986014 } } }, "c_eval": { "prompt_1": { "accuracy": 0.4903417533432392 }, "prompt_2": { "accuracy": 0.5044576523031203 }, "prompt_3": { "accuracy": 0.49257057949479943 }, "prompt_4": { "accuracy": 0.4390787518573551 }, "prompt_5": { "accuracy": 0.4576523031203566 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.4919053549190536, "category_acc": { "computer_network": 0.7916666666666666, "operating_system": 0.6666666666666666, "computer_architecture": 0.5, "college_programming": 0.6428571428571429, "college_physics": 0.4583333333333333, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.7692307692307693, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.56, "veterinary_medicine": 0.5, "college_economics": 0.4, "business_administration": 0.5, "marxism": 0.5, "mao_zedong_thought": 0.6551724137931034, "education_science": 0.5588235294117647, "teacher_qualification": 0.7142857142857143, "high_school_politics": 0.625, "high_school_geography": 0.5416666666666666, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.7058823529411765, "modern_chinese_history": 0.6071428571428571, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.4444444444444444, "law": 0.3448275862068966, "chinese_language_and_literature": 0.42857142857142855, "art_studies": 0.631578947368421, "professional_tour_guide": 0.5588235294117647, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.52, "middle_school_history": 0.7407407407407407, "civil_servant": 0.4423076923076923, "sports_science": 0.4166666666666667, "plant_protection": 0.7407407407407407, "basic_medicine": 0.5, "clinical_medicine": 0.48148148148148145, "urban_and_rural_planner": 0.49019607843137253, "accountant": 0.3888888888888889, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.4166666666666667, "tax_accountant": 0.3148148148148148, "physician": 0.6296296296296297 } }, "prompt_2": { "accuracy": 0.48443337484433374, "category_acc": { "computer_network": 0.7083333333333334, "operating_system": 0.7083333333333334, "computer_architecture": 0.6153846153846154, "college_programming": 0.47619047619047616, "college_physics": 0.4166666666666667, "college_chemistry": 0.41379310344827586, "advanced_mathematics": 0.375, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.4166666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.25, "middle_school_biology": 0.7307692307692307, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.5, "college_economics": 0.4666666666666667, "business_administration": 0.4473684210526316, "marxism": 0.5833333333333334, "mao_zedong_thought": 0.6206896551724138, "education_science": 0.6176470588235294, "teacher_qualification": 0.7142857142857143, "high_school_politics": 0.625, "high_school_geography": 0.5416666666666666, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.5357142857142857, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.48148148148148145, "law": 0.4482758620689655, "chinese_language_and_literature": 0.4642857142857143, "art_studies": 0.6052631578947368, "professional_tour_guide": 0.5294117647058824, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.4, "middle_school_history": 0.6296296296296297, "civil_servant": 0.4423076923076923, "sports_science": 0.5416666666666666, "plant_protection": 0.7037037037037037, "basic_medicine": 0.5, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.5686274509803921, "accountant": 0.42592592592592593, "fire_engineer": 0.4166666666666667, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.3148148148148148, "physician": 0.5740740740740741 } }, "prompt_3": { "accuracy": 0.4919053549190536, "category_acc": { "computer_network": 0.5, "operating_system": 0.625, "computer_architecture": 0.5, "college_programming": 0.6666666666666666, "college_physics": 0.4166666666666667, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.42857142857142855, "electrical_engineer": 0.38095238095238093, "metrology_engineer": 0.5862068965517241, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.7692307692307693, "middle_school_physics": 0.5, "middle_school_chemistry": 0.52, "veterinary_medicine": 0.4642857142857143, "college_economics": 0.45, "business_administration": 0.3684210526315789, "marxism": 0.5833333333333334, "mao_zedong_thought": 0.5862068965517241, "education_science": 0.6470588235294118, "teacher_qualification": 0.7346938775510204, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.5833333333333334, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.5294117647058824, "modern_chinese_history": 0.5357142857142857, "ideological_and_moral_cultivation": 0.75, "logic": 0.48148148148148145, "law": 0.3793103448275862, "chinese_language_and_literature": 0.42857142857142855, "art_studies": 0.5526315789473685, "professional_tour_guide": 0.5882352941176471, "legal_professional": 0.5, "high_school_chinese": 0.25, "high_school_history": 0.56, "middle_school_history": 0.6666666666666666, "civil_servant": 0.3076923076923077, "sports_science": 0.5, "plant_protection": 0.7407407407407407, "basic_medicine": 0.5, "clinical_medicine": 0.48148148148148145, "urban_and_rural_planner": 0.5882352941176471, "accountant": 0.3888888888888889, "fire_engineer": 0.4722222222222222, "environmental_impact_assessment_engineer": 0.3888888888888889, "tax_accountant": 0.25925925925925924, "physician": 0.6111111111111112 } }, "prompt_4": { "accuracy": 0.41594022415940224, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.3333333333333333, "computer_architecture": 0.5384615384615384, "college_programming": 0.35714285714285715, "college_physics": 0.3333333333333333, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.38095238095238093, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.5769230769230769, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.4642857142857143, "college_economics": 0.36666666666666664, "business_administration": 0.42105263157894735, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.5, "teacher_qualification": 0.5918367346938775, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.6153846153846154, "middle_school_geography": 0.5294117647058824, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.5, "logic": 0.5185185185185185, "law": 0.41379310344827586, "chinese_language_and_literature": 0.4642857142857143, "art_studies": 0.6052631578947368, "professional_tour_guide": 0.47058823529411764, "legal_professional": 0.35714285714285715, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.64, "middle_school_history": 0.37037037037037035, "civil_servant": 0.4230769230769231, "sports_science": 0.375, "plant_protection": 0.5555555555555556, "basic_medicine": 0.2916666666666667, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.49019607843137253, "accountant": 0.3333333333333333, "fire_engineer": 0.3333333333333333, "environmental_impact_assessment_engineer": 0.4722222222222222, "tax_accountant": 0.25925925925925924, "physician": 0.5370370370370371 } }, "prompt_5": { "accuracy": 0.4701120797011208, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.4583333333333333, "computer_architecture": 0.46153846153846156, "college_programming": 0.35714285714285715, "college_physics": 0.4166666666666667, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.4827586206896552, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.375, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.375, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.7307692307692307, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.52, "veterinary_medicine": 0.42857142857142855, "college_economics": 0.36666666666666664, "business_administration": 0.47368421052631576, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.5862068965517241, "education_science": 0.6470588235294118, "teacher_qualification": 0.673469387755102, "high_school_politics": 0.6666666666666666, "high_school_geography": 0.5, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.5882352941176471, "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.5185185185185185, "law": 0.3793103448275862, "chinese_language_and_literature": 0.4642857142857143, "art_studies": 0.5526315789473685, "professional_tour_guide": 0.5294117647058824, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.16666666666666666, "high_school_history": 0.4, "middle_school_history": 0.5925925925925926, "civil_servant": 0.36538461538461536, "sports_science": 0.5, "plant_protection": 0.6296296296296297, "basic_medicine": 0.5833333333333334, "clinical_medicine": 0.5185185185185185, "urban_and_rural_planner": 0.5490196078431373, "accountant": 0.48148148148148145, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.4444444444444444, "tax_accountant": 0.25925925925925924, "physician": 0.6296296296296297 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.5806451612903226 }, "prompt_2": { "accuracy": 0.5448028673835126 }, "prompt_3": { "accuracy": 0.5232974910394266 }, "prompt_4": { "accuracy": 0.4731182795698925 }, "prompt_5": { "accuracy": 0.5161290322580645 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.5118286997064411, "category_acc": { "agronomy": 0.4911242603550296, "anatomy": 0.4189189189189189, "ancient_chinese": 0.32926829268292684, "arts": 0.6, "astronomy": 0.3212121212121212, "business_ethics": 0.5263157894736842, "chinese_civil_service_exam": 0.43125, "chinese_driving_rule": 0.6717557251908397, "chinese_food_culture": 0.4411764705882353, "chinese_foreign_policy": 0.5607476635514018, "chinese_history": 0.5139318885448917, "chinese_literature": 0.3382352941176471, "chinese_teacher_qualification": 0.5865921787709497, "clinical_knowledge": 0.5021097046413502, "college_actuarial_science": 0.2641509433962264, "college_education": 0.6448598130841121, "college_engineering_hydrology": 0.5, "college_law": 0.3888888888888889, "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.5283018867924528, "college_medicine": 0.5384615384615384, "computer_science": 0.6323529411764706, "computer_security": 0.672514619883041, "conceptual_physics": 0.5102040816326531, "construction_project_management": 0.4028776978417266, "economics": 0.6352201257861635, "education": 0.5337423312883436, "electrical_engineering": 0.5058139534883721, "elementary_chinese": 0.3492063492063492, "elementary_commonsense": 0.47474747474747475, "elementary_information_and_technology": 0.7521008403361344, "elementary_mathematics": 0.3826086956521739, "ethnology": 0.48148148148148145, "food_science": 0.5384615384615384, "genetics": 0.42613636363636365, "global_facts": 0.5771812080536913, "high_school_biology": 0.42011834319526625, "high_school_chemistry": 0.32575757575757575, "high_school_geography": 0.4745762711864407, "high_school_mathematics": 0.2865853658536585, "high_school_physics": 0.4090909090909091, "high_school_politics": 0.4965034965034965, "human_sexuality": 0.5714285714285714, "international_law": 0.4810810810810811, "journalism": 0.5523255813953488, "jurisprudence": 0.5036496350364964, "legal_and_moral_basis": 0.8130841121495327, "logical": 0.6097560975609756, "machine_learning": 0.4426229508196721, "management": 0.6285714285714286, "marketing": 0.6055555555555555, "marxist_theory": 0.6084656084656085, "modern_chinese": 0.43103448275862066, "nutrition": 0.5310344827586206, "philosophy": 0.6476190476190476, "professional_accounting": 0.5885714285714285, "professional_law": 0.33175355450236965, "professional_medicine": 0.4654255319148936, "professional_psychology": 0.5732758620689655, "public_relations": 0.5862068965517241, "security_study": 0.5703703703703704, "sociology": 0.6061946902654868, "sports_science": 0.5636363636363636, "traditional_chinese_medicine": 0.41621621621621624, "virology": 0.6094674556213018, "world_history": 0.5217391304347826, "world_religions": 0.6375 } }, "prompt_2": { "accuracy": 0.49378345708858573, "category_acc": { "agronomy": 0.5384615384615384, "anatomy": 0.36486486486486486, "ancient_chinese": 0.36585365853658536, "arts": 0.56875, "astronomy": 0.2909090909090909, "business_ethics": 0.5215311004784688, "chinese_civil_service_exam": 0.3875, "chinese_driving_rule": 0.6870229007633588, "chinese_food_culture": 0.4117647058823529, "chinese_foreign_policy": 0.5794392523364486, "chinese_history": 0.4891640866873065, "chinese_literature": 0.3627450980392157, "chinese_teacher_qualification": 0.5754189944134078, "clinical_knowledge": 0.5021097046413502, "college_actuarial_science": 0.20754716981132076, "college_education": 0.6261682242990654, "college_engineering_hydrology": 0.49056603773584906, "college_law": 0.3888888888888889, "college_mathematics": 0.3333333333333333, "college_medical_statistics": 0.6037735849056604, "college_medicine": 0.5347985347985348, "computer_science": 0.5980392156862745, "computer_security": 0.6023391812865497, "conceptual_physics": 0.3197278911564626, "construction_project_management": 0.41007194244604317, "economics": 0.5786163522012578, "education": 0.5337423312883436, "electrical_engineering": 0.5116279069767442, "elementary_chinese": 0.3611111111111111, "elementary_commonsense": 0.4444444444444444, "elementary_information_and_technology": 0.7142857142857143, "elementary_mathematics": 0.41304347826086957, "ethnology": 0.4888888888888889, "food_science": 0.48951048951048953, "genetics": 0.4034090909090909, "global_facts": 0.5704697986577181, "high_school_biology": 0.31952662721893493, "high_school_chemistry": 0.29545454545454547, "high_school_geography": 0.5, "high_school_mathematics": 0.2073170731707317, "high_school_physics": 0.34545454545454546, "high_school_politics": 0.4755244755244755, "human_sexuality": 0.5079365079365079, "international_law": 0.43243243243243246, "journalism": 0.5465116279069767, "jurisprudence": 0.5060827250608273, "legal_and_moral_basis": 0.822429906542056, "logical": 0.5203252032520326, "machine_learning": 0.4098360655737705, "management": 0.6095238095238096, "marketing": 0.5944444444444444, "marxist_theory": 0.6084656084656085, "modern_chinese": 0.3879310344827586, "nutrition": 0.4827586206896552, "philosophy": 0.5714285714285714, "professional_accounting": 0.5428571428571428, "professional_law": 0.3459715639810427, "professional_medicine": 0.4441489361702128, "professional_psychology": 0.5732758620689655, "public_relations": 0.5459770114942529, "security_study": 0.5185185185185185, "sociology": 0.584070796460177, "sports_science": 0.5818181818181818, "traditional_chinese_medicine": 0.4486486486486487, "virology": 0.5798816568047337, "world_history": 0.546583850931677, "world_religions": 0.61875 } }, "prompt_3": { "accuracy": 0.5107926092212053, "category_acc": { "agronomy": 0.5325443786982249, "anatomy": 0.3918918918918919, "ancient_chinese": 0.3353658536585366, "arts": 0.575, "astronomy": 0.3212121212121212, "business_ethics": 0.5550239234449761, "chinese_civil_service_exam": 0.44375, "chinese_driving_rule": 0.648854961832061, "chinese_food_culture": 0.375, "chinese_foreign_policy": 0.6074766355140186, "chinese_history": 0.5201238390092879, "chinese_literature": 0.35294117647058826, "chinese_teacher_qualification": 0.5977653631284916, "clinical_knowledge": 0.5569620253164557, "college_actuarial_science": 0.29245283018867924, "college_education": 0.6261682242990654, "college_engineering_hydrology": 0.5283018867924528, "college_law": 0.3888888888888889, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.5471698113207547, "college_medicine": 0.5347985347985348, "computer_science": 0.6421568627450981, "computer_security": 0.6608187134502924, "conceptual_physics": 0.4421768707482993, "construction_project_management": 0.37410071942446044, "economics": 0.6226415094339622, "education": 0.5398773006134969, "electrical_engineering": 0.5058139534883721, "elementary_chinese": 0.34523809523809523, "elementary_commonsense": 0.45454545454545453, "elementary_information_and_technology": 0.7605042016806722, "elementary_mathematics": 0.4391304347826087, "ethnology": 0.48148148148148145, "food_science": 0.5314685314685315, "genetics": 0.4318181818181818, "global_facts": 0.5906040268456376, "high_school_biology": 0.34911242603550297, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.5084745762711864, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.32727272727272727, "high_school_politics": 0.48951048951048953, "human_sexuality": 0.5555555555555556, "international_law": 0.4486486486486487, "journalism": 0.5581395348837209, "jurisprudence": 0.48905109489051096, "legal_and_moral_basis": 0.8364485981308412, "logical": 0.5528455284552846, "machine_learning": 0.4262295081967213, "management": 0.6047619047619047, "marketing": 0.6444444444444445, "marxist_theory": 0.6031746031746031, "modern_chinese": 0.3620689655172414, "nutrition": 0.5103448275862069, "philosophy": 0.6666666666666666, "professional_accounting": 0.56, "professional_law": 0.3744075829383886, "professional_medicine": 0.4734042553191489, "professional_psychology": 0.5732758620689655, "public_relations": 0.5574712643678161, "security_study": 0.6222222222222222, "sociology": 0.5707964601769911, "sports_science": 0.5757575757575758, "traditional_chinese_medicine": 0.4486486486486487, "virology": 0.6094674556213018, "world_history": 0.5714285714285714, "world_religions": 0.64375 } }, "prompt_4": { "accuracy": 0.4285960973925056, "category_acc": { "agronomy": 0.4319526627218935, "anatomy": 0.3310810810810811, "ancient_chinese": 0.3048780487804878, "arts": 0.5125, "astronomy": 0.296969696969697, "business_ethics": 0.40669856459330145, "chinese_civil_service_exam": 0.4125, "chinese_driving_rule": 0.5725190839694656, "chinese_food_culture": 0.3602941176470588, "chinese_foreign_policy": 0.4953271028037383, "chinese_history": 0.4582043343653251, "chinese_literature": 0.3284313725490196, "chinese_teacher_qualification": 0.48044692737430167, "clinical_knowledge": 0.5274261603375527, "college_actuarial_science": 0.2358490566037736, "college_education": 0.5514018691588785, "college_engineering_hydrology": 0.5094339622641509, "college_law": 0.3425925925925926, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.3584905660377358, "college_medicine": 0.4652014652014652, "computer_science": 0.4950980392156863, "computer_security": 0.4853801169590643, "conceptual_physics": 0.24489795918367346, "construction_project_management": 0.3597122302158273, "economics": 0.46540880503144655, "education": 0.4294478527607362, "electrical_engineering": 0.4127906976744186, "elementary_chinese": 0.35714285714285715, "elementary_commonsense": 0.398989898989899, "elementary_information_and_technology": 0.5672268907563025, "elementary_mathematics": 0.3173913043478261, "ethnology": 0.4074074074074074, "food_science": 0.44755244755244755, "genetics": 0.30113636363636365, "global_facts": 0.4966442953020134, "high_school_biology": 0.3254437869822485, "high_school_chemistry": 0.2803030303030303, "high_school_geography": 0.3898305084745763, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.3090909090909091, "high_school_politics": 0.44755244755244755, "human_sexuality": 0.48412698412698413, "international_law": 0.43243243243243246, "journalism": 0.4418604651162791, "jurisprudence": 0.39659367396593675, "legal_and_moral_basis": 0.7102803738317757, "logical": 0.4065040650406504, "machine_learning": 0.36885245901639346, "management": 0.5333333333333333, "marketing": 0.55, "marxist_theory": 0.5026455026455027, "modern_chinese": 0.3017241379310345, "nutrition": 0.4, "philosophy": 0.5619047619047619, "professional_accounting": 0.5085714285714286, "professional_law": 0.3175355450236967, "professional_medicine": 0.4095744680851064, "professional_psychology": 0.49137931034482757, "public_relations": 0.4827586206896552, "security_study": 0.5259259259259259, "sociology": 0.4823008849557522, "sports_science": 0.41818181818181815, "traditional_chinese_medicine": 0.372972972972973, "virology": 0.4319526627218935, "world_history": 0.5031055900621118, "world_religions": 0.53125 } }, "prompt_5": { "accuracy": 0.4709894664134001, "category_acc": { "agronomy": 0.47337278106508873, "anatomy": 0.33783783783783783, "ancient_chinese": 0.2926829268292683, "arts": 0.54375, "astronomy": 0.3212121212121212, "business_ethics": 0.49760765550239233, "chinese_civil_service_exam": 0.38125, "chinese_driving_rule": 0.6793893129770993, "chinese_food_culture": 0.36764705882352944, "chinese_foreign_policy": 0.5327102803738317, "chinese_history": 0.4953560371517028, "chinese_literature": 0.3382352941176471, "chinese_teacher_qualification": 0.5642458100558659, "clinical_knowledge": 0.510548523206751, "college_actuarial_science": 0.24528301886792453, "college_education": 0.5607476635514018, "college_engineering_hydrology": 0.5094339622641509, "college_law": 0.37962962962962965, "college_mathematics": 0.21904761904761905, "college_medical_statistics": 0.5094339622641509, "college_medicine": 0.47619047619047616, "computer_science": 0.5735294117647058, "computer_security": 0.5497076023391813, "conceptual_physics": 0.3401360544217687, "construction_project_management": 0.3597122302158273, "economics": 0.559748427672956, "education": 0.5460122699386503, "electrical_engineering": 0.4883720930232558, "elementary_chinese": 0.3531746031746032, "elementary_commonsense": 0.4292929292929293, "elementary_information_and_technology": 0.6848739495798319, "elementary_mathematics": 0.3652173913043478, "ethnology": 0.4888888888888889, "food_science": 0.5524475524475524, "genetics": 0.3977272727272727, "global_facts": 0.5369127516778524, "high_school_biology": 0.33136094674556216, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.4661016949152542, "high_school_mathematics": 0.29878048780487804, "high_school_physics": 0.3181818181818182, "high_school_politics": 0.4965034965034965, "human_sexuality": 0.5158730158730159, "international_law": 0.42162162162162165, "journalism": 0.5116279069767442, "jurisprudence": 0.44525547445255476, "legal_and_moral_basis": 0.8084112149532711, "logical": 0.5121951219512195, "machine_learning": 0.4344262295081967, "management": 0.6285714285714286, "marketing": 0.5722222222222222, "marxist_theory": 0.582010582010582, "modern_chinese": 0.3275862068965517, "nutrition": 0.46206896551724136, "philosophy": 0.6095238095238096, "professional_accounting": 0.4514285714285714, "professional_law": 0.32701421800947866, "professional_medicine": 0.4122340425531915, "professional_psychology": 0.5172413793103449, "public_relations": 0.5459770114942529, "security_study": 0.5407407407407407, "sociology": 0.5132743362831859, "sports_science": 0.509090909090909, "traditional_chinese_medicine": 0.4, "virology": 0.5088757396449705, "world_history": 0.5341614906832298, "world_religions": 0.5875 } } }, "zbench": { "prompt_1": { "accuracy": 0.30303030303030304 }, "prompt_2": { "accuracy": 0.2727272727272727 }, "prompt_3": { "accuracy": 0.42424242424242425 }, "prompt_4": { "accuracy": 0.2727272727272727 }, "prompt_5": { "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6863636363636364 }, "prompt_2": { "accuracy": 0.6477272727272727 }, "prompt_3": { "accuracy": 0.6795454545454546 }, "prompt_4": { "accuracy": 0.6590909090909091 }, "prompt_5": { "accuracy": 0.6363636363636364 } }, "ocnli": { "prompt_1": { "accuracy": 0.49627118644067797 }, "prompt_2": { "accuracy": 0.4505084745762712 }, "prompt_3": { "accuracy": 0.5345762711864407 }, "prompt_4": { "accuracy": 0.48033898305084743 }, "prompt_5": { "accuracy": 0.42338983050847456 } }, "c3": { "prompt_1": { "accuracy": 0.8844427823485415 }, "prompt_2": { "accuracy": 0.8825729244577412 }, "prompt_3": { "accuracy": 0.8739715781600599 }, "prompt_4": { "accuracy": 0.8754674644727001 }, "prompt_5": { "accuracy": 0.8694839192221391 } }, "dream": { "prompt_1": { "accuracy": 0.8931896129348359 }, "prompt_2": { "accuracy": 0.8990690837824595 }, "prompt_3": { "accuracy": 0.9118079372856442 }, "prompt_4": { "accuracy": 0.9024987751102401 }, "prompt_5": { "accuracy": 0.9064184223419892 } }, "samsum": { "prompt_1": { "rouge1": 0.22783552386864417, "rouge2": 0.0839403119084876, "rougeL": 0.17007006250391818, "avg_rouge": 0.16061529942701666 }, "prompt_2": { "rouge1": 0.21906277397329468, "rouge2": 0.07981495779273613, "rougeL": 0.1611019992992204, "avg_rouge": 0.15332657702175043 }, "prompt_3": { "rouge1": 0.22051601719774955, "rouge2": 0.07733715420256924, "rougeL": 0.16309052127918097, "avg_rouge": 0.15364789755983324 }, "prompt_4": { "rouge1": 0.21865452524714854, "rouge2": 0.07795006249760242, "rougeL": 0.16086961877960224, "avg_rouge": 0.1524914021747844 }, "prompt_5": { "rouge1": 0.22042637368342233, "rouge2": 0.08018824424560042, "rougeL": 0.1712332946359059, "avg_rouge": 0.15728263752164287 } }, "dialogsum": { "prompt_1": { "rouge1": 0.12705281041724492, "rouge2": 0.034748531923445534, "rougeL": 0.09916061070001848, "avg_rouge": 0.08698731768023632 }, "prompt_2": { "rouge1": 0.13768897790409756, "rouge2": 0.037895101154647844, "rougeL": 0.10580890122208217, "avg_rouge": 0.0937976600936092 }, "prompt_3": { "rouge1": 0.15013721929677626, "rouge2": 0.043554485551585684, "rougeL": 0.11271506493749443, "avg_rouge": 0.1021355899286188 }, "prompt_4": { "rouge1": 0.16301602801162368, "rouge2": 0.04606017775287866, "rougeL": 0.12228466252459767, "avg_rouge": 0.11045362276303333 }, "prompt_5": { "rouge1": 0.19426426170381578, "rouge2": 0.05026998096295629, "rougeL": 0.1427788616737507, "avg_rouge": 0.1291043681135076 } }, "sst2": { "prompt_1": { "accuracy": 0.9025229357798165 }, "prompt_2": { "accuracy": 0.911697247706422 }, "prompt_3": { "accuracy": 0.8692660550458715 }, "prompt_4": { "accuracy": 0.9128440366972477 }, "prompt_5": { "accuracy": 0.8990825688073395 } }, "cola": { "prompt_1": { "accuracy": 0.6663470757430489 }, "prompt_2": { "accuracy": 0.6452540747842761 }, "prompt_3": { "accuracy": 0.6903163950143816 }, "prompt_4": { "accuracy": 0.7948226270373921 }, "prompt_5": { "accuracy": 0.6586768935762224 } }, "qqp": { "prompt_1": { "accuracy": 0.639 }, "prompt_2": { "accuracy": 0.7385 }, "prompt_3": { "accuracy": 0.6695 }, "prompt_4": { "accuracy": 0.5695 }, "prompt_5": { "accuracy": 0.75 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.6 } }, "qnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "wnli": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.7 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "flan_t5_base": { "model_size": "0.25B", "model_link": "https://huggingface.co/google/flan-t5-base", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.7214285714285714, "language_acc": { "Spanish": 0.8546218487394958, "English": 0.9495798319327731, "Chinese": 0.3521008403361345, "Vietnamese": 0.7294117647058823 }, "consistency_score_2": 0.5616246498599441, "consistency_score_3": 0.3859243697478991, "consistency_score_4": 0.2605042016806723, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8630252100840337, "Spanish,Chinese": 0.35042016806722687, "Spanish,Vietnamese": 0.7218487394957983, "English,Chinese": 0.35042016806722687, "English,Vietnamese": 0.7260504201680672, "Chinese,Vietnamese": 0.35798319327731093 }, "3_combine": { "Spanish,English,Chinese": 0.3142857142857143, "Spanish,English,Vietnamese": 0.6756302521008404, "Spanish,Chinese,Vietnamese": 0.2773109243697479, "English,Chinese,Vietnamese": 0.27647058823529413 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.2605042016806723 } }, "AC3_2": 0.6315748436203689, "AC3_3": 0.5028511802673522, "AC3_4": 0.38278623383839344 }, "prompt_2": { "overall_acc": 0.7195378151260504, "language_acc": { "Spanish": 0.8554621848739495, "English": 0.946218487394958, "Chinese": 0.34705882352941175, "Vietnamese": 0.7294117647058823 }, "consistency_score_2": 0.5628851540616246, "consistency_score_3": 0.38886554621848735, "consistency_score_4": 0.2689075630252101, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8563025210084033, "Spanish,Chinese": 0.34705882352941175, "Spanish,Vietnamese": 0.726890756302521, "English,Chinese": 0.346218487394958, "English,Vietnamese": 0.7243697478991596, "Chinese,Vietnamese": 0.3764705882352941 }, "3_combine": { "Spanish,English,Chinese": 0.31092436974789917, "Spanish,English,Vietnamese": 0.673109243697479, "Spanish,Chinese,Vietnamese": 0.2865546218487395, "English,Chinese,Vietnamese": 0.2848739495798319 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.2689075630252101 } }, "AC3_2": 0.6316436364912575, "AC3_3": 0.5048766094306893, "AC3_4": 0.3915019780098478 }, "prompt_3": { "overall_acc": 0.7153361344537815, "language_acc": { "Spanish": 0.8495798319327731, "English": 0.9453781512605042, "Chinese": 0.34789915966386553, "Vietnamese": 0.7184873949579832 }, "consistency_score_2": 0.5570028011204482, "consistency_score_3": 0.3779411764705882, "consistency_score_4": 0.25630252100840334, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8579831932773109, "Spanish,Chinese": 0.34369747899159664, "Spanish,Vietnamese": 0.7151260504201681, "English,Chinese": 0.3445378151260504, "English,Vietnamese": 0.7126050420168067, "Chinese,Vietnamese": 0.3680672268907563 }, "3_combine": { "Spanish,English,Chinese": 0.3050420168067227, "Spanish,English,Vietnamese": 0.6638655462184874, "Spanish,Chinese,Vietnamese": 0.2680672268907563, "English,Chinese,Vietnamese": 0.2747899159663866 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.25630252100840334 } }, "AC3_2": 0.6263177514445691, "AC3_3": 0.49457713519010127, "AC3_4": 0.37738814440812757 }, "prompt_4": { "overall_acc": 0.7256302521008403, "language_acc": { "Spanish": 0.8588235294117647, "English": 0.9487394957983193, "Chinese": 0.3680672268907563, "Vietnamese": 0.726890756302521 }, "consistency_score_2": 0.5682072829131654, "consistency_score_3": 0.39768907563025213, "consistency_score_4": 0.2747899159663866, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8638655462184874, "Spanish,Chinese": 0.36722689075630255, "Spanish,Vietnamese": 0.7243697478991596, "English,Chinese": 0.36554621848739494, "English,Vietnamese": 0.726890756302521, "Chinese,Vietnamese": 0.36134453781512604 }, "3_combine": { "Spanish,English,Chinese": 0.3319327731092437, "Spanish,English,Vietnamese": 0.6773109243697479, "Spanish,Chinese,Vietnamese": 0.2915966386554622, "English,Chinese,Vietnamese": 0.28991596638655465 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.2747899159663866 } }, "AC3_2": 0.6373418342813549, "AC3_3": 0.5137901878079704, "AC3_4": 0.3986242627690158 }, "prompt_5": { "overall_acc": 0.7016806722689075, "language_acc": { "Spanish": 0.8428571428571429, "English": 0.9394957983193277, "Chinese": 0.3302521008403361, "Vietnamese": 0.6941176470588235 }, "consistency_score_2": 0.5323529411764707, "consistency_score_3": 0.34936974789915964, "consistency_score_4": 0.22941176470588234, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8403361344537815, "Spanish,Chinese": 0.32857142857142857, "Spanish,Vietnamese": 0.6865546218487395, "English,Chinese": 0.32521008403361346, "English,Vietnamese": 0.6865546218487395, "Chinese,Vietnamese": 0.326890756302521 }, "3_combine": { "Spanish,English,Chinese": 0.2815126050420168, "Spanish,English,Vietnamese": 0.6277310924369748, "Spanish,Chinese,Vietnamese": 0.24369747899159663, "English,Chinese,Vietnamese": 0.2445378151260504 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.22941176470588234 } }, "AC3_2": 0.605399667478831, "AC3_3": 0.46647809629109394, "AC3_4": 0.34577404965494263 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.31428571428571433, "language_acc": { "Filipino": 0.2866666666666667, "Vietnamese": 0.3, "Chinese": 0.30666666666666664, "Spanish": 0.34, "Malay": 0.2866666666666667, "Indonesian": 0.26666666666666666, "English": 0.41333333333333333 }, "consistency_score_2": 0.4257142857142857, "consistency_score_3": 0.23733333333333329, "consistency_score_4": 0.15314285714285714, "consistency_score_5": 0.10603174603174605, "consistency_score_6": 0.07523809523809523, "consistency_score_7": 0.05333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.44, "Filipino,Chinese": 0.38666666666666666, "Filipino,Spanish": 0.43333333333333335, "Filipino,Malay": 0.5933333333333334, "Filipino,Indonesian": 0.5, "Filipino,English": 0.52, "Vietnamese,Chinese": 0.36666666666666664, "Vietnamese,Spanish": 0.44, "Vietnamese,Malay": 0.46, "Vietnamese,Indonesian": 0.4066666666666667, "Vietnamese,English": 0.36666666666666664, "Chinese,Spanish": 0.25333333333333335, "Chinese,Malay": 0.37333333333333335, "Chinese,Indonesian": 0.32, "Chinese,English": 0.32666666666666666, "Spanish,Malay": 0.41333333333333333, "Spanish,Indonesian": 0.44, "Spanish,English": 0.48, "Malay,Indonesian": 0.58, "Malay,English": 0.42, "Indonesian,English": 0.42 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.20666666666666667, "Filipino,Vietnamese,Spanish": 0.28, "Filipino,Vietnamese,Malay": 0.32666666666666666, "Filipino,Vietnamese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,English": 0.26666666666666666, "Filipino,Chinese,Spanish": 0.14666666666666667, "Filipino,Chinese,Malay": 0.25333333333333335, "Filipino,Chinese,Indonesian": 0.19333333333333333, "Filipino,Chinese,English": 0.19333333333333333, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.26666666666666666, "Filipino,Spanish,English": 0.30666666666666664, "Filipino,Malay,Indonesian": 0.38666666666666666, "Filipino,Malay,English": 0.3333333333333333, "Filipino,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish": 0.17333333333333334, "Vietnamese,Chinese,Malay": 0.21333333333333335, "Vietnamese,Chinese,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,English": 0.14666666666666667, "Vietnamese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,English": 0.26, "Vietnamese,Malay,Indonesian": 0.29333333333333333, "Vietnamese,Malay,English": 0.24, "Vietnamese,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay": 0.14, "Chinese,Spanish,Indonesian": 0.13333333333333333, "Chinese,Spanish,English": 0.14, "Chinese,Malay,Indonesian": 0.21333333333333335, "Chinese,Malay,English": 0.16, "Chinese,Indonesian,English": 0.14666666666666667, "Spanish,Malay,Indonesian": 0.29333333333333333, "Spanish,Malay,English": 0.26, "Spanish,Indonesian,English": 0.26, "Malay,Indonesian,English": 0.29333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.12, "Filipino,Vietnamese,Chinese,Malay": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,English": 0.12, "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.2, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay": 0.11333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.08666666666666667, "Filipino,Chinese,Spanish,English": 0.1, "Filipino,Chinese,Malay,Indonesian": 0.14666666666666667, "Filipino,Chinese,Malay,English": 0.14, "Filipino,Chinese,Indonesian,English": 0.11333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.22, "Filipino,Spanish,Malay,English": 0.22, "Filipino,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay": 0.12, "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Vietnamese,Chinese,Spanish,English": 0.10666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.14, "Vietnamese,Chinese,Malay,English": 0.11333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, "Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Vietnamese,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian": 0.1, "Chinese,Spanish,Malay,English": 0.09333333333333334, "Chinese,Spanish,Indonesian,English": 0.08666666666666667, "Chinese,Malay,Indonesian,English": 0.10666666666666667, "Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08, "Filipino,Vietnamese,Chinese,Spanish,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.08, "Filipino,Chinese,Spanish,Indonesian,English": 0.06666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.16, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 } }, "AC3_2": 0.36161059013315255, "AC3_3": 0.27044198890124926, "AC3_4": 0.20593782740966227, "AC3_5": 0.15856711260793696, "AC3_6": 0.12141110719900902, "AC3_7": 0.09119170981975355 }, "prompt_2": { "overall_acc": 0.31238095238095237, "language_acc": { "Filipino": 0.3, "Vietnamese": 0.3, "Chinese": 0.24, "Spanish": 0.3333333333333333, "Malay": 0.30666666666666664, "Indonesian": 0.29333333333333333, "English": 0.41333333333333333 }, "consistency_score_2": 0.4152380952380952, "consistency_score_3": 0.22780952380952385, "consistency_score_4": 0.1459047619047619, "consistency_score_5": 0.10317460317460314, "consistency_score_6": 0.07714285714285715, "consistency_score_7": 0.06, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4266666666666667, "Filipino,Chinese": 0.29333333333333333, "Filipino,Spanish": 0.47333333333333333, "Filipino,Malay": 0.5333333333333333, "Filipino,Indonesian": 0.47333333333333333, "Filipino,English": 0.5, "Vietnamese,Chinese": 0.26666666666666666, "Vietnamese,Spanish": 0.44666666666666666, "Vietnamese,Malay": 0.5133333333333333, "Vietnamese,Indonesian": 0.44, "Vietnamese,English": 0.36666666666666664, "Chinese,Spanish": 0.29333333333333333, "Chinese,Malay": 0.30666666666666664, "Chinese,Indonesian": 0.32666666666666666, "Chinese,English": 0.25333333333333335, "Spanish,Malay": 0.44666666666666666, "Spanish,Indonesian": 0.4866666666666667, "Spanish,English": 0.47333333333333333, "Malay,Indonesian": 0.6333333333333333, "Malay,English": 0.38, "Indonesian,English": 0.38666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.12, "Filipino,Vietnamese,Spanish": 0.26666666666666666, "Filipino,Vietnamese,Malay": 0.32, "Filipino,Vietnamese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,English": 0.24, "Filipino,Chinese,Spanish": 0.15333333333333332, "Filipino,Chinese,Malay": 0.17333333333333334, "Filipino,Chinese,Indonesian": 0.18, "Filipino,Chinese,English": 0.14666666666666667, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.29333333333333333, "Filipino,Spanish,English": 0.31333333333333335, "Filipino,Malay,Indonesian": 0.37333333333333335, "Filipino,Malay,English": 0.29333333333333333, "Filipino,Indonesian,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish": 0.13333333333333333, "Vietnamese,Chinese,Malay": 0.18, "Vietnamese,Chinese,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,English": 0.09333333333333334, "Vietnamese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,English": 0.26666666666666666, "Vietnamese,Malay,Indonesian": 0.3466666666666667, "Vietnamese,Malay,English": 0.24, "Vietnamese,Indonesian,English": 0.22, "Chinese,Spanish,Malay": 0.14, "Chinese,Spanish,Indonesian": 0.17333333333333334, "Chinese,Spanish,English": 0.14, "Chinese,Malay,Indonesian": 0.2, "Chinese,Malay,English": 0.12666666666666668, "Chinese,Indonesian,English": 0.12, "Spanish,Malay,Indonesian": 0.34, "Spanish,Malay,English": 0.25333333333333335, "Spanish,Indonesian,English": 0.26, "Malay,Indonesian,English": 0.2866666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.1, "Filipino,Vietnamese,Chinese,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.2, "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Malay,English": 0.2, "Filipino,Vietnamese,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay": 0.1, "Filipino,Chinese,Spanish,Indonesian": 0.12, "Filipino,Chinese,Spanish,English": 0.08666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.14666666666666667, "Filipino,Chinese,Malay,English": 0.10666666666666667, "Filipino,Chinese,Indonesian,English": 0.09333333333333334, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.2, "Filipino,Spanish,Indonesian,English": 0.2, "Filipino,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.10666666666666667, "Vietnamese,Chinese,Spanish,English": 0.07333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Vietnamese,Chinese,Malay,English": 0.08666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.08666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.18, "Vietnamese,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian": 0.12, "Chinese,Spanish,Malay,English": 0.08, "Chinese,Spanish,Indonesian,English": 0.09333333333333334, "Chinese,Malay,Indonesian,English": 0.1, "Spanish,Malay,Indonesian,English": 0.22 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08, "Filipino,Vietnamese,Chinese,Spanish,English": 0.06, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.06666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.06666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.06, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 } }, "AC3_2": 0.3565395162810914, "AC3_3": 0.263475048644892, "AC3_4": 0.1989059097852009, "AC3_5": 0.15511659205405445, "AC3_6": 0.12373035274504304, "AC3_7": 0.10066496160979585 }, "prompt_3": { "overall_acc": 0.32285714285714284, "language_acc": { "Filipino": 0.29333333333333333, "Vietnamese": 0.32666666666666666, "Chinese": 0.28, "Spanish": 0.3466666666666667, "Malay": 0.32666666666666666, "Indonesian": 0.2866666666666667, "English": 0.4 }, "consistency_score_2": 0.4403174603174603, "consistency_score_3": 0.2582857142857144, "consistency_score_4": 0.1746666666666667, "consistency_score_5": 0.12730158730158725, "consistency_score_6": 0.09619047619047619, "consistency_score_7": 0.07333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.41333333333333333, "Filipino,Chinese": 0.35333333333333333, "Filipino,Spanish": 0.46, "Filipino,Malay": 0.5266666666666666, "Filipino,Indonesian": 0.4666666666666667, "Filipino,English": 0.5466666666666666, "Vietnamese,Chinese": 0.32666666666666666, "Vietnamese,Spanish": 0.46, "Vietnamese,Malay": 0.5266666666666666, "Vietnamese,Indonesian": 0.4533333333333333, "Vietnamese,English": 0.36666666666666664, "Chinese,Spanish": 0.3466666666666667, "Chinese,Malay": 0.36666666666666664, "Chinese,Indonesian": 0.37333333333333335, "Chinese,English": 0.35333333333333333, "Spanish,Malay": 0.4266666666666667, "Spanish,Indonesian": 0.44666666666666666, "Spanish,English": 0.5066666666666667, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.44666666666666666, "Indonesian,English": 0.43333333333333335 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.16666666666666666, "Filipino,Vietnamese,Spanish": 0.28, "Filipino,Vietnamese,Malay": 0.32, "Filipino,Vietnamese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,English": 0.28, "Filipino,Chinese,Spanish": 0.20666666666666667, "Filipino,Chinese,Malay": 0.21333333333333335, "Filipino,Chinese,Indonesian": 0.20666666666666667, "Filipino,Chinese,English": 0.24, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.28, "Filipino,Spanish,English": 0.3466666666666667, "Filipino,Malay,Indonesian": 0.37333333333333335, "Filipino,Malay,English": 0.34, "Filipino,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish": 0.16666666666666666, "Vietnamese,Chinese,Malay": 0.2, "Vietnamese,Chinese,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,English": 0.16666666666666666, "Vietnamese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Spanish,Indonesian": 0.29333333333333333, "Vietnamese,Spanish,English": 0.28, "Vietnamese,Malay,Indonesian": 0.36, "Vietnamese,Malay,English": 0.2733333333333333, "Vietnamese,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay": 0.19333333333333333, "Chinese,Spanish,Indonesian": 0.2, "Chinese,Spanish,English": 0.22666666666666666, "Chinese,Malay,Indonesian": 0.24, "Chinese,Malay,English": 0.20666666666666667, "Chinese,Indonesian,English": 0.19333333333333333, "Spanish,Malay,Indonesian": 0.32666666666666666, "Spanish,Malay,English": 0.26666666666666666, "Spanish,Indonesian,English": 0.28, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,English": 0.14, "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Malay,English": 0.24, "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay": 0.14, "Filipino,Chinese,Spanish,Indonesian": 0.14, "Filipino,Chinese,Spanish,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.16666666666666666, "Filipino,Chinese,Malay,English": 0.17333333333333334, "Filipino,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.23333333333333334, "Filipino,Spanish,Indonesian,English": 0.21333333333333335, "Filipino,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay": 0.12, "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Vietnamese,Chinese,Spanish,English": 0.12, "Vietnamese,Chinese,Malay,Indonesian": 0.14, "Vietnamese,Chinese,Malay,English": 0.13333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.12, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Chinese,Spanish,Malay,English": 0.14, "Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Chinese,Malay,Indonesian,English": 0.14666666666666667, "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.12, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 } }, "AC3_2": 0.3725481340134615, "AC3_3": 0.28698412693474434, "AC3_4": 0.22669218984723838, "AC3_5": 0.18260326411418085, "AC3_6": 0.14822077918540835, "AC3_7": 0.11951923073906365 }, "prompt_4": { "overall_acc": 0.3295238095238095, "language_acc": { "Filipino": 0.30666666666666664, "Vietnamese": 0.3333333333333333, "Chinese": 0.30666666666666664, "Spanish": 0.3466666666666667, "Malay": 0.30666666666666664, "Indonesian": 0.30666666666666664, "English": 0.4 }, "consistency_score_2": 0.46126984126984116, "consistency_score_3": 0.27999999999999997, "consistency_score_4": 0.19980952380952377, "consistency_score_5": 0.15746031746031747, "consistency_score_6": 0.13142857142857142, "consistency_score_7": 0.11333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.44666666666666666, "Filipino,Chinese": 0.38666666666666666, "Filipino,Spanish": 0.47333333333333333, "Filipino,Malay": 0.5933333333333334, "Filipino,Indonesian": 0.5, "Filipino,English": 0.52, "Vietnamese,Chinese": 0.42, "Vietnamese,Spanish": 0.4533333333333333, "Vietnamese,Malay": 0.5333333333333333, "Vietnamese,Indonesian": 0.4866666666666667, "Vietnamese,English": 0.38666666666666666, "Chinese,Spanish": 0.30666666666666664, "Chinese,Malay": 0.4533333333333333, "Chinese,Indonesian": 0.4066666666666667, "Chinese,English": 0.36666666666666664, "Spanish,Malay": 0.44, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.5066666666666667, "Malay,Indonesian": 0.6666666666666666, "Malay,English": 0.42, "Indonesian,English": 0.4266666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.22666666666666666, "Filipino,Vietnamese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Malay": 0.36, "Filipino,Vietnamese,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,English": 0.26, "Filipino,Chinese,Spanish": 0.18666666666666668, "Filipino,Chinese,Malay": 0.29333333333333333, "Filipino,Chinese,Indonesian": 0.24, "Filipino,Chinese,English": 0.22, "Filipino,Spanish,Malay": 0.31333333333333335, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.42, "Filipino,Malay,English": 0.3333333333333333, "Filipino,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish": 0.21333333333333335, "Vietnamese,Chinese,Malay": 0.3, "Vietnamese,Chinese,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,English": 0.18666666666666668, "Vietnamese,Spanish,Malay": 0.32, "Vietnamese,Spanish,Indonesian": 0.31333333333333335, "Vietnamese,Spanish,English": 0.2866666666666667, "Vietnamese,Malay,Indonesian": 0.4, "Vietnamese,Malay,English": 0.26666666666666666, "Vietnamese,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay": 0.2, "Chinese,Spanish,Indonesian": 0.20666666666666667, "Chinese,Spanish,English": 0.19333333333333333, "Chinese,Malay,Indonesian": 0.31333333333333335, "Chinese,Malay,English": 0.2, "Chinese,Indonesian,English": 0.21333333333333335, "Spanish,Malay,Indonesian": 0.36, "Spanish,Malay,English": 0.29333333333333333, "Spanish,Indonesian,English": 0.29333333333333333, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,English": 0.14, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Malay,English": 0.17333333333333334, "Filipino,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Spanish,Malay,English": 0.25333333333333335, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, "Vietnamese,Spanish,Malay,English": 0.22666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, "Vietnamese,Malay,Indonesian,English": 0.22, "Chinese,Spanish,Malay,Indonesian": 0.18, "Chinese,Spanish,Malay,English": 0.15333333333333332, "Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Chinese,Malay,Indonesian,English": 0.18, "Spanish,Malay,Indonesian,English": 0.25333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,English": 0.12, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,Malay,English": 0.2, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 } }, "AC3_2": 0.3844223967704124, "AC3_3": 0.30274999995033003, "AC3_4": 0.24877328256579573, "AC3_5": 0.21309492762744078, "AC3_6": 0.18791027150586953, "AC3_7": 0.16865949816980086 }, "prompt_5": { "overall_acc": 0.30190476190476195, "language_acc": { "Filipino": 0.3, "Vietnamese": 0.26666666666666666, "Chinese": 0.32, "Spanish": 0.29333333333333333, "Malay": 0.29333333333333333, "Indonesian": 0.26666666666666666, "English": 0.37333333333333335 }, "consistency_score_2": 0.4374603174603175, "consistency_score_3": 0.2500952380952381, "consistency_score_4": 0.1622857142857143, "consistency_score_5": 0.11174603174603175, "consistency_score_6": 0.07809523809523809, "consistency_score_7": 0.05333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4533333333333333, "Filipino,Chinese": 0.36, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.5133333333333333, "Filipino,Indonesian": 0.49333333333333335, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.43333333333333335, "Vietnamese,Spanish": 0.3933333333333333, "Vietnamese,Malay": 0.5266666666666666, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.32, "Chinese,Spanish": 0.30666666666666664, "Chinese,Malay": 0.4, "Chinese,Indonesian": 0.4, "Chinese,English": 0.24666666666666667, "Spanish,Malay": 0.4533333333333333, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.5133333333333333, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.43333333333333335, "Indonesian,English": 0.38666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.21333333333333335, "Filipino,Vietnamese,Spanish": 0.26, "Filipino,Vietnamese,Malay": 0.34, "Filipino,Vietnamese,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,English": 0.24, "Filipino,Chinese,Spanish": 0.18, "Filipino,Chinese,Malay": 0.22, "Filipino,Chinese,Indonesian": 0.22, "Filipino,Chinese,English": 0.16666666666666666, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.29333333333333333, "Filipino,Malay,Indonesian": 0.36, "Filipino,Malay,English": 0.3, "Filipino,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish": 0.18, "Vietnamese,Chinese,Malay": 0.2733333333333333, "Vietnamese,Chinese,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,English": 0.12, "Vietnamese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,English": 0.22666666666666666, "Vietnamese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Malay,English": 0.23333333333333334, "Vietnamese,Indonesian,English": 0.22, "Chinese,Spanish,Malay": 0.17333333333333334, "Chinese,Spanish,Indonesian": 0.2, "Chinese,Spanish,English": 0.14, "Chinese,Malay,Indonesian": 0.28, "Chinese,Malay,English": 0.15333333333333332, "Chinese,Indonesian,English": 0.14666666666666667, "Spanish,Malay,Indonesian": 0.36, "Spanish,Malay,English": 0.30666666666666664, "Spanish,Indonesian,English": 0.2733333333333333, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,English": 0.1, "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Spanish,English": 0.18, "Filipino,Vietnamese,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Malay,English": 0.2, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.12, "Filipino,Chinese,Spanish,Indonesian": 0.14, "Filipino,Chinese,Spanish,English": 0.11333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.16, "Filipino,Chinese,Malay,English": 0.12, "Filipino,Chinese,Indonesian,English": 0.11333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Spanish,Malay,English": 0.23333333333333334, "Filipino,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, "Vietnamese,Chinese,Spanish,English": 0.08666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.2, "Vietnamese,Chinese,Malay,English": 0.10666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, "Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Malay,Indonesian,English": 0.2, "Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Chinese,Spanish,Malay,English": 0.1, "Chinese,Spanish,Indonesian,English": 0.1, "Chinese,Malay,Indonesian,English": 0.12666666666666668, "Spanish,Malay,Indonesian,English": 0.24 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.1, "Filipino,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.08666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.06666666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 } }, "AC3_2": 0.35725612868054457, "AC3_3": 0.2735686351339825, "AC3_4": 0.21109795411361834, "AC3_5": 0.16311661729053772, "AC3_6": 0.12409118029912626, "AC3_7": 0.09065236816036149 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.275974025974026, "language_acc": { "Indonesian": 0.25, "English": 0.42045454545454547, "Filipino": 0.26704545454545453, "Spanish": 0.2897727272727273, "Chinese": 0.23295454545454544, "Malay": 0.24431818181818182, "Vietnamese": 0.22727272727272727 }, "consistency_score_2": 0.3533549783549783, "consistency_score_3": 0.15454545454545449, "consistency_score_4": 0.07581168831168832, "consistency_score_5": 0.03923160173160173, "consistency_score_6": 0.022727272727272724, "consistency_score_7": 0.017045454545454544, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.26704545454545453, "Indonesian,Filipino": 0.4943181818181818, "Indonesian,Spanish": 0.4318181818181818, "Indonesian,Chinese": 0.26704545454545453, "Indonesian,Malay": 0.5397727272727273, "Indonesian,Vietnamese": 0.4375, "English,Filipino": 0.3522727272727273, "English,Spanish": 0.3693181818181818, "English,Chinese": 0.25, "English,Malay": 0.29545454545454547, "English,Vietnamese": 0.24431818181818182, "Filipino,Spanish": 0.4431818181818182, "Filipino,Chinese": 0.26136363636363635, "Filipino,Malay": 0.4772727272727273, "Filipino,Vietnamese": 0.45454545454545453, "Spanish,Chinese": 0.2556818181818182, "Spanish,Malay": 0.3693181818181818, "Spanish,Vietnamese": 0.3181818181818182, "Chinese,Malay": 0.24431818181818182, "Chinese,Vietnamese": 0.18181818181818182, "Malay,Vietnamese": 0.4659090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.1590909090909091, "Indonesian,English,Spanish": 0.1590909090909091, "Indonesian,English,Chinese": 0.07386363636363637, "Indonesian,English,Malay": 0.14772727272727273, "Indonesian,English,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish": 0.2727272727272727, "Indonesian,Filipino,Chinese": 0.13636363636363635, "Indonesian,Filipino,Malay": 0.3125, "Indonesian,Filipino,Vietnamese": 0.2727272727272727, "Indonesian,Spanish,Chinese": 0.11931818181818182, "Indonesian,Spanish,Malay": 0.24431818181818182, "Indonesian,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Chinese,Malay": 0.1534090909090909, "Indonesian,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Malay,Vietnamese": 0.2840909090909091, "English,Filipino,Spanish": 0.19886363636363635, "English,Filipino,Chinese": 0.10227272727272728, "English,Filipino,Malay": 0.17613636363636365, "English,Filipino,Vietnamese": 0.13636363636363635, "English,Spanish,Chinese": 0.08522727272727272, "English,Spanish,Malay": 0.14204545454545456, "English,Spanish,Vietnamese": 0.10227272727272728, "English,Chinese,Malay": 0.09659090909090909, "English,Chinese,Vietnamese": 0.05113636363636364, "English,Malay,Vietnamese": 0.13068181818181818, "Filipino,Spanish,Chinese": 0.11363636363636363, "Filipino,Spanish,Malay": 0.2215909090909091, "Filipino,Spanish,Vietnamese": 0.20454545454545456, "Filipino,Chinese,Malay": 0.13068181818181818, "Filipino,Chinese,Vietnamese": 0.09090909090909091, "Filipino,Malay,Vietnamese": 0.2784090909090909, "Spanish,Chinese,Malay": 0.10227272727272728, "Spanish,Chinese,Vietnamese": 0.07954545454545454, "Spanish,Malay,Vietnamese": 0.17613636363636365, "Chinese,Malay,Vietnamese": 0.07954545454545454 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.10795454545454546, "Indonesian,English,Filipino,Chinese": 0.045454545454545456, "Indonesian,English,Filipino,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Vietnamese": 0.07954545454545454, "Indonesian,English,Spanish,Chinese": 0.045454545454545456, "Indonesian,English,Spanish,Malay": 0.07954545454545454, "Indonesian,English,Spanish,Vietnamese": 0.06818181818181818, "Indonesian,English,Chinese,Malay": 0.05113636363636364, "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Malay,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Spanish,Chinese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay": 0.09090909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese": 0.045454545454545456, "English,Filipino,Spanish,Malay": 0.10227272727272728, "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Malay,Vietnamese": 0.08522727272727272, "English,Spanish,Chinese,Malay": 0.045454545454545456, "English,Spanish,Chinese,Vietnamese": 0.022727272727272728, "English,Spanish,Malay,Vietnamese": 0.056818181818181816, "English,Chinese,Malay,Vietnamese": 0.028409090909090908, "Filipino,Spanish,Chinese,Malay": 0.0625, "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.0625, "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.028409090909090908, "Indonesian,English,Filipino,Spanish,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay": 0.03409090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.045454545454545456, "Indonesian,English,Spanish,Chinese,Malay": 0.03409090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Spanish,Malay,Vietnamese": 0.03409090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 } }, "AC3_2": 0.309907203667074, "AC3_3": 0.19813519808917573, "AC3_4": 0.1189477342807884, "AC3_5": 0.0686973970017828, "AC3_6": 0.041996047416770495, "AC3_7": 0.03210778140430308 }, "prompt_2": { "overall_acc": 0.2775974025974026, "language_acc": { "Indonesian": 0.22727272727272727, "English": 0.4147727272727273, "Filipino": 0.2556818181818182, "Spanish": 0.2840909090909091, "Chinese": 0.2897727272727273, "Malay": 0.24431818181818182, "Vietnamese": 0.22727272727272727 }, "consistency_score_2": 0.35633116883116883, "consistency_score_3": 0.15665584415584413, "consistency_score_4": 0.07499999999999998, "consistency_score_5": 0.03733766233766234, "consistency_score_6": 0.01948051948051948, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.29545454545454547, "Indonesian,Filipino": 0.48295454545454547, "Indonesian,Spanish": 0.45454545454545453, "Indonesian,Chinese": 0.19886363636363635, "Indonesian,Malay": 0.6079545454545454, "Indonesian,Vietnamese": 0.48295454545454547, "English,Filipino": 0.36363636363636365, "English,Spanish": 0.375, "English,Chinese": 0.24431818181818182, "English,Malay": 0.26704545454545453, "English,Vietnamese": 0.24431818181818182, "Filipino,Spanish": 0.4090909090909091, "Filipino,Chinese": 0.19318181818181818, "Filipino,Malay": 0.48295454545454547, "Filipino,Vietnamese": 0.5738636363636364, "Spanish,Chinese": 0.2159090909090909, "Spanish,Malay": 0.375, "Spanish,Vietnamese": 0.3352272727272727, "Chinese,Malay": 0.18181818181818182, "Chinese,Vietnamese": 0.19318181818181818, "Malay,Vietnamese": 0.5056818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.18181818181818182, "Indonesian,English,Spanish": 0.18181818181818182, "Indonesian,English,Chinese": 0.056818181818181816, "Indonesian,English,Malay": 0.17045454545454544, "Indonesian,English,Vietnamese": 0.125, "Indonesian,Filipino,Spanish": 0.25, "Indonesian,Filipino,Chinese": 0.08522727272727272, "Indonesian,Filipino,Malay": 0.3409090909090909, "Indonesian,Filipino,Vietnamese": 0.3181818181818182, "Indonesian,Spanish,Chinese": 0.09090909090909091, "Indonesian,Spanish,Malay": 0.2784090909090909, "Indonesian,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,Chinese,Malay": 0.10795454545454546, "Indonesian,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Malay,Vietnamese": 0.3409090909090909, "English,Filipino,Spanish": 0.17613636363636365, "English,Filipino,Chinese": 0.09090909090909091, "English,Filipino,Malay": 0.16477272727272727, "English,Filipino,Vietnamese": 0.17045454545454544, "English,Spanish,Chinese": 0.08522727272727272, "English,Spanish,Malay": 0.13636363636363635, "English,Spanish,Vietnamese": 0.11931818181818182, "English,Chinese,Malay": 0.0625, "English,Chinese,Vietnamese": 0.05113636363636364, "English,Malay,Vietnamese": 0.11931818181818182, "Filipino,Spanish,Chinese": 0.08522727272727272, "Filipino,Spanish,Malay": 0.2159090909090909, "Filipino,Spanish,Vietnamese": 0.23863636363636365, "Filipino,Chinese,Malay": 0.07386363636363637, "Filipino,Chinese,Vietnamese": 0.10227272727272728, "Filipino,Malay,Vietnamese": 0.3409090909090909, "Spanish,Chinese,Malay": 0.06818181818181818, "Spanish,Chinese,Vietnamese": 0.08522727272727272, "Spanish,Malay,Vietnamese": 0.19318181818181818, "Chinese,Malay,Vietnamese": 0.07954545454545454 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.11363636363636363, "Indonesian,English,Filipino,Chinese": 0.03977272727272727, "Indonesian,English,Filipino,Malay": 0.11931818181818182, "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese": 0.03409090909090909, "Indonesian,English,Spanish,Malay": 0.10227272727272728, "Indonesian,English,Spanish,Vietnamese": 0.08522727272727272, "Indonesian,English,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Malay,Vietnamese": 0.07954545454545454, "Indonesian,Filipino,Spanish,Chinese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Chinese,Malay": 0.05113636363636364, "Indonesian,Filipino,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.056818181818181816, "English,Filipino,Spanish,Chinese": 0.045454545454545456, "English,Filipino,Spanish,Malay": 0.09659090909090909, "English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "English,Filipino,Chinese,Malay": 0.03977272727272727, "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Malay,Vietnamese": 0.10227272727272728, "English,Spanish,Chinese,Malay": 0.017045454545454544, "English,Spanish,Chinese,Vietnamese": 0.03409090909090909, "English,Spanish,Malay,Vietnamese": 0.06818181818181818, "English,Chinese,Malay,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Malay": 0.07386363636363637, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Malay,Vietnamese": 0.05113636363636364, "Indonesian,English,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, "AC3_2": 0.3120749288630882, "AC3_3": 0.2002852287437799, "AC3_4": 0.11809392261844114, "AC3_5": 0.06582206451250455, "AC3_6": 0.036406216721830816, "AC3_7": 0.021833503567520746 }, "prompt_3": { "overall_acc": 0.27272727272727276, "language_acc": { "Indonesian": 0.2215909090909091, "English": 0.4034090909090909, "Filipino": 0.26136363636363635, "Spanish": 0.30113636363636365, "Chinese": 0.23295454545454544, "Malay": 0.23863636363636365, "Vietnamese": 0.25 }, "consistency_score_2": 0.3566017316017316, "consistency_score_3": 0.15584415584415584, "consistency_score_4": 0.07483766233766233, "consistency_score_5": 0.036525974025974024, "consistency_score_6": 0.01704545454545454, "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.29545454545454547, "Indonesian,Filipino": 0.4772727272727273, "Indonesian,Spanish": 0.4318181818181818, "Indonesian,Chinese": 0.17613636363636365, "Indonesian,Malay": 0.5738636363636364, "Indonesian,Vietnamese": 0.5, "English,Filipino": 0.3465909090909091, "English,Spanish": 0.4147727272727273, "English,Chinese": 0.22727272727272727, "English,Malay": 0.2784090909090909, "English,Vietnamese": 0.23863636363636365, "Filipino,Spanish": 0.4147727272727273, "Filipino,Chinese": 0.24431818181818182, "Filipino,Malay": 0.48863636363636365, "Filipino,Vietnamese": 0.5340909090909091, "Spanish,Chinese": 0.2556818181818182, "Spanish,Malay": 0.36363636363636365, "Spanish,Vietnamese": 0.3125, "Chinese,Malay": 0.17613636363636365, "Chinese,Vietnamese": 0.19886363636363635, "Malay,Vietnamese": 0.5397727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.18181818181818182, "Indonesian,English,Spanish": 0.18181818181818182, "Indonesian,English,Chinese": 0.03977272727272727, "Indonesian,English,Malay": 0.17613636363636365, "Indonesian,English,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish": 0.24431818181818182, "Indonesian,Filipino,Chinese": 0.08522727272727272, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.3125, "Indonesian,Spanish,Chinese": 0.07954545454545454, "Indonesian,Spanish,Malay": 0.2556818181818182, "Indonesian,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,Chinese,Malay": 0.06818181818181818, "Indonesian,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish": 0.19318181818181818, "English,Filipino,Chinese": 0.07954545454545454, "English,Filipino,Malay": 0.17045454545454544, "English,Filipino,Vietnamese": 0.16477272727272727, "English,Spanish,Chinese": 0.09090909090909091, "English,Spanish,Malay": 0.1590909090909091, "English,Spanish,Vietnamese": 0.125, "English,Chinese,Malay": 0.056818181818181816, "English,Chinese,Vietnamese": 0.045454545454545456, "English,Malay,Vietnamese": 0.13636363636363635, "Filipino,Spanish,Chinese": 0.125, "Filipino,Spanish,Malay": 0.2159090909090909, "Filipino,Spanish,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay": 0.09090909090909091, "Filipino,Chinese,Vietnamese": 0.10795454545454546, "Filipino,Malay,Vietnamese": 0.3409090909090909, "Spanish,Chinese,Malay": 0.0625, "Spanish,Chinese,Vietnamese": 0.07954545454545454, "Spanish,Malay,Vietnamese": 0.19886363636363635, "Chinese,Malay,Vietnamese": 0.07954545454545454 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.125, "Indonesian,English,Filipino,Chinese": 0.028409090909090908, "Indonesian,English,Filipino,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese": 0.022727272727272728, "Indonesian,English,Spanish,Malay": 0.10795454545454546, "Indonesian,English,Spanish,Vietnamese": 0.08522727272727272, "Indonesian,English,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.05113636363636364, "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay": 0.05113636363636364, "Indonesian,Filipino,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,Filipino,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Chinese,Malay": 0.03409090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.03977272727272727, "English,Filipino,Spanish,Chinese": 0.045454545454545456, "English,Filipino,Spanish,Malay": 0.11363636363636363, "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, "English,Filipino,Chinese,Malay": 0.03409090909090909, "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.022727272727272728, "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, "English,Spanish,Malay,Vietnamese": 0.08522727272727272, "English,Chinese,Malay,Vietnamese": 0.017045454545454544, "Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Malay": 0.07954545454545454, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay": 0.011363636363636364, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Spanish,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, "AC3_2": 0.3090752754921998, "AC3_3": 0.19834710739173553, "AC3_4": 0.11744724212153103, "AC3_5": 0.06442376519033474, "AC3_6": 0.03208556148625353, "AC3_7": 0.01113172541344137 }, "prompt_4": { "overall_acc": 0.27516233766233766, "language_acc": { "Indonesian": 0.25, "English": 0.4090909090909091, "Filipino": 0.2556818181818182, "Spanish": 0.2784090909090909, "Chinese": 0.24431818181818182, "Malay": 0.2556818181818182, "Vietnamese": 0.23295454545454544 }, "consistency_score_2": 0.36093073593073594, "consistency_score_3": 0.15941558441558443, "consistency_score_4": 0.07499999999999997, "consistency_score_5": 0.034902597402597414, "consistency_score_6": 0.01542207792207792, "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.30113636363636365, "Indonesian,Filipino": 0.4943181818181818, "Indonesian,Spanish": 0.48863636363636365, "Indonesian,Chinese": 0.26704545454545453, "Indonesian,Malay": 0.5568181818181818, "Indonesian,Vietnamese": 0.48863636363636365, "English,Filipino": 0.35795454545454547, "English,Spanish": 0.39204545454545453, "English,Chinese": 0.26704545454545453, "English,Malay": 0.2727272727272727, "English,Vietnamese": 0.25, "Filipino,Spanish": 0.3977272727272727, "Filipino,Chinese": 0.17613636363636365, "Filipino,Malay": 0.5, "Filipino,Vietnamese": 0.5113636363636364, "Spanish,Chinese": 0.2556818181818182, "Spanish,Malay": 0.4318181818181818, "Spanish,Vietnamese": 0.3522727272727273, "Chinese,Malay": 0.19886363636363635, "Chinese,Vietnamese": 0.11931818181818182, "Malay,Vietnamese": 0.5 }, "3_combine": { "Indonesian,English,Filipino": 0.17045454545454544, "Indonesian,English,Spanish": 0.19318181818181818, "Indonesian,English,Chinese": 0.10227272727272728, "Indonesian,English,Malay": 0.1534090909090909, "Indonesian,English,Vietnamese": 0.125, "Indonesian,Filipino,Spanish": 0.26704545454545453, "Indonesian,Filipino,Chinese": 0.09659090909090909, "Indonesian,Filipino,Malay": 0.3352272727272727, "Indonesian,Filipino,Vietnamese": 0.3068181818181818, "Indonesian,Spanish,Chinese": 0.14772727272727273, "Indonesian,Spanish,Malay": 0.30113636363636365, "Indonesian,Spanish,Vietnamese": 0.23295454545454544, "Indonesian,Chinese,Malay": 0.13068181818181818, "Indonesian,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,Malay,Vietnamese": 0.32386363636363635, "English,Filipino,Spanish": 0.17613636363636365, "English,Filipino,Chinese": 0.08522727272727272, "English,Filipino,Malay": 0.16477272727272727, "English,Filipino,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese": 0.10227272727272728, "English,Spanish,Malay": 0.1590909090909091, "English,Spanish,Vietnamese": 0.125, "English,Chinese,Malay": 0.07386363636363637, "English,Chinese,Vietnamese": 0.03977272727272727, "English,Malay,Vietnamese": 0.11363636363636363, "Filipino,Spanish,Chinese": 0.07386363636363637, "Filipino,Spanish,Malay": 0.23863636363636365, "Filipino,Spanish,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay": 0.07386363636363637, "Filipino,Chinese,Vietnamese": 0.0625, "Filipino,Malay,Vietnamese": 0.3125, "Spanish,Chinese,Malay": 0.11363636363636363, "Spanish,Chinese,Vietnamese": 0.07954545454545454, "Spanish,Malay,Vietnamese": 0.2215909090909091, "Chinese,Malay,Vietnamese": 0.045454545454545456 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.11363636363636363, "Indonesian,English,Filipino,Chinese": 0.05113636363636364, "Indonesian,English,Filipino,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese": 0.06818181818181818, "Indonesian,English,Spanish,Malay": 0.10227272727272728, "Indonesian,English,Spanish,Vietnamese": 0.07954545454545454, "Indonesian,English,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Malay,Vietnamese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Chinese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay": 0.0625, "Indonesian,Filipino,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Chinese": 0.03409090909090909, "English,Filipino,Spanish,Malay": 0.10227272727272728, "English,Filipino,Spanish,Vietnamese": 0.08522727272727272, "English,Filipino,Chinese,Malay": 0.03977272727272727, "English,Filipino,Chinese,Vietnamese": 0.017045454545454544, "English,Filipino,Malay,Vietnamese": 0.07954545454545454, "English,Spanish,Chinese,Malay": 0.03977272727272727, "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, "English,Spanish,Malay,Vietnamese": 0.06818181818181818, "English,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.028409090909090908, "Indonesian,English,Filipino,Spanish,Malay": 0.07386363636363637, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.05113636363636364, "Indonesian,English,Spanish,Chinese,Malay": 0.03409090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Malay,Vietnamese": 0.045454545454545456, "Indonesian,English,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, "AC3_2": 0.312264192585157, "AC3_3": 0.20187479680406284, "AC3_4": 0.11787204447259654, "AC3_5": 0.06194754196683012, "AC3_6": 0.029207175496006994, "AC3_7": 0.01113373620202616 }, "prompt_5": { "overall_acc": 0.2824675324675325, "language_acc": { "Indonesian": 0.25, "English": 0.4034090909090909, "Filipino": 0.2840909090909091, "Spanish": 0.26704545454545453, "Chinese": 0.2727272727272727, "Malay": 0.26704545454545453, "Vietnamese": 0.23295454545454544 }, "consistency_score_2": 0.36688311688311687, "consistency_score_3": 0.16818181818181818, "consistency_score_4": 0.08571428571428572, "consistency_score_5": 0.04518398268398267, "consistency_score_6": 0.023538961038961036, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.2840909090909091, "Indonesian,Filipino": 0.45454545454545453, "Indonesian,Spanish": 0.44886363636363635, "Indonesian,Chinese": 0.26136363636363635, "Indonesian,Malay": 0.5397727272727273, "Indonesian,Vietnamese": 0.3977272727272727, "English,Filipino": 0.3125, "English,Spanish": 0.4090909090909091, "English,Chinese": 0.29545454545454547, "English,Malay": 0.26136363636363635, "English,Vietnamese": 0.26704545454545453, "Filipino,Spanish": 0.4943181818181818, "Filipino,Chinese": 0.22727272727272727, "Filipino,Malay": 0.5511363636363636, "Filipino,Vietnamese": 0.44886363636363635, "Spanish,Chinese": 0.2159090909090909, "Spanish,Malay": 0.42045454545454547, "Spanish,Vietnamese": 0.3465909090909091, "Chinese,Malay": 0.26704545454545453, "Chinese,Vietnamese": 0.30113636363636365, "Malay,Vietnamese": 0.5 }, "3_combine": { "Indonesian,English,Filipino": 0.1534090909090909, "Indonesian,English,Spanish": 0.1875, "Indonesian,English,Chinese": 0.09090909090909091, "Indonesian,English,Malay": 0.14204545454545456, "Indonesian,English,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish": 0.2727272727272727, "Indonesian,Filipino,Chinese": 0.10227272727272728, "Indonesian,Filipino,Malay": 0.3125, "Indonesian,Filipino,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Chinese": 0.07954545454545454, "Indonesian,Spanish,Malay": 0.26136363636363635, "Indonesian,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Chinese,Malay": 0.125, "Indonesian,Chinese,Vietnamese": 0.125, "Indonesian,Malay,Vietnamese": 0.2840909090909091, "English,Filipino,Spanish": 0.20454545454545456, "English,Filipino,Chinese": 0.07954545454545454, "English,Filipino,Malay": 0.17613636363636365, "English,Filipino,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese": 0.11931818181818182, "English,Spanish,Malay": 0.17613636363636365, "English,Spanish,Vietnamese": 0.14204545454545456, "English,Chinese,Malay": 0.10227272727272728, "English,Chinese,Vietnamese": 0.10227272727272728, "English,Malay,Vietnamese": 0.14204545454545456, "Filipino,Spanish,Chinese": 0.10227272727272728, "Filipino,Spanish,Malay": 0.30113636363636365, "Filipino,Spanish,Vietnamese": 0.23295454545454544, "Filipino,Chinese,Malay": 0.14204545454545456, "Filipino,Chinese,Vietnamese": 0.13068181818181818, "Filipino,Malay,Vietnamese": 0.32954545454545453, "Spanish,Chinese,Malay": 0.09659090909090909, "Spanish,Chinese,Vietnamese": 0.10795454545454546, "Spanish,Malay,Vietnamese": 0.2159090909090909, "Chinese,Malay,Vietnamese": 0.16477272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.10227272727272728, "Indonesian,English,Filipino,Chinese": 0.03409090909090909, "Indonesian,English,Filipino,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese": 0.05113636363636364, "Indonesian,English,Spanish,Malay": 0.10227272727272728, "Indonesian,English,Spanish,Vietnamese": 0.07386363636363637, "Indonesian,English,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Malay,Vietnamese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Chinese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Chinese,Malay": 0.08522727272727272, "Indonesian,Filipino,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Spanish,Chinese,Malay": 0.05113636363636364, "Indonesian,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese": 0.045454545454545456, "English,Filipino,Spanish,Malay": 0.13068181818181818, "English,Filipino,Spanish,Vietnamese": 0.08522727272727272, "English,Filipino,Chinese,Malay": 0.06818181818181818, "English,Filipino,Chinese,Vietnamese": 0.056818181818181816, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.056818181818181816, "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, "English,Spanish,Malay,Vietnamese": 0.08522727272727272, "English,Chinese,Malay,Vietnamese": 0.06818181818181818, "Filipino,Spanish,Chinese,Malay": 0.06818181818181818, "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Malay": 0.06818181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.045454545454545456, "Indonesian,English,Filipino,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Filipino,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Malay,Vietnamese": 0.05113636363636364, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.03977272727272727, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, "AC3_2": 0.3191883116391567, "AC3_3": 0.21083311496500506, "AC3_4": 0.13151927434069596, "AC3_5": 0.0779059915898941, "AC3_6": 0.04345654344234227, "AC3_7": 0.021848317420992257 } }, "sg_eval": { "prompt_1": { "accuracy": 0.4174757281553398 }, "prompt_2": { "accuracy": 0.42718446601941745 }, "prompt_3": { "accuracy": 0.39805825242718446 }, "prompt_4": { "accuracy": 0.3883495145631068 }, "prompt_5": { "accuracy": 0.4077669902912621 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2571428571428571 }, "prompt_2": { "accuracy": 0.24761904761904763 }, "prompt_3": { "accuracy": 0.24761904761904763 }, "prompt_4": { "accuracy": 0.23809523809523808 }, "prompt_5": { "accuracy": 0.24761904761904763 } }, "us_eval": { "prompt_1": { "accuracy": 0.2616822429906542 }, "prompt_2": { "accuracy": 0.2897196261682243 }, "prompt_3": { "accuracy": 0.2897196261682243 }, "prompt_4": { "accuracy": 0.29906542056074764 }, "prompt_5": { "accuracy": 0.3177570093457944 } }, "ph_eval": { "prompt_1": { "accuracy": 0.18, "category_acc": { "brand": 0.1, "demographics": 0.0, "biology": 0.1, "history": 0.3333333333333333, "literature": 0.2, "politics": 0.2, "culture": 0.2, "film": 0.2, "law": 0.1, "geography": 0.2 } }, "prompt_2": { "accuracy": 0.2, "category_acc": { "brand": 0.1, "demographics": 0.0, "biology": 0.2, "history": 0.2, "literature": 0.2, "politics": 0.2, "culture": 0.2, "film": 0.3, "law": 0.1, "geography": 0.4 } }, "prompt_3": { "accuracy": 0.22, "category_acc": { "brand": 0.1, "demographics": 0.2, "biology": 0.2, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.2, "culture": 0.2, "film": 0.2, "law": 0.2, "geography": 0.4 } }, "prompt_4": { "accuracy": 0.2, "category_acc": { "brand": 0.1, "demographics": 0.0, "biology": 0.1, "history": 0.3333333333333333, "literature": 0.2, "politics": 0.3, "culture": 0.3, "film": 0.2, "law": 0.1, "geography": 0.2 } }, "prompt_5": { "accuracy": 0.22, "category_acc": { "brand": 0.1, "demographics": 0.4, "biology": 0.1, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.2, "culture": 0.2, "film": 0.2, "law": 0.2, "geography": 0.4 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.17678014707244064 }, "prompt_2": { "bleu_score": 0.17731433484914794 }, "prompt_3": { "bleu_score": 0.17952240698146069 }, "prompt_4": { "bleu_score": 0.1839610854412572 }, "prompt_5": { "bleu_score": 0.16307560288744835 } }, "indommlu": { "prompt_1": { "accuracy": 0.23946858935843515, "category_acc": { "History": 0.21887550200803213, "Geography": 0.21428571428571427, "Lampungic": 0.2925170068027211, "Social science": 0.2287145242070117, "Balinese": 0.267515923566879, "Makassarese": 0.21505376344086022, "Banjarese": 0.2013888888888889, "Chemistry": 0.1927007299270073, "Biology": 0.23550295857988165, "Science": 0.2579979360165119, "Christian religion": 0.24875621890547264, "Art": 0.23627287853577372, "Islam religion": 0.21621621621621623, "Hindu religion": 0.2, "Madurese": 0.23389830508474577, "Sport": 0.20270270270270271, "Indonesian language": 0.2534246575342466, "Physics": 0.21212121212121213, "Minangkabau culture": 0.22110552763819097, "Dayak language": 0.29357798165137616, "Sociology": 0.2217741935483871, "Economy": 0.21311475409836064, "Sundanese": 0.2515125324114088, "Javanese": 0.25806451612903225, "Civic education": 0.26895565092989987 } }, "prompt_2": { "accuracy": 0.2373990253020896, "category_acc": { "History": 0.20883534136546184, "Geography": 0.20816326530612245, "Lampungic": 0.29931972789115646, "Social science": 0.24707846410684475, "Balinese": 0.25690021231422505, "Makassarese": 0.16666666666666666, "Banjarese": 0.2222222222222222, "Chemistry": 0.16934306569343066, "Biology": 0.22011834319526627, "Science": 0.23426212590299278, "Christian religion": 0.2935323383084577, "Art": 0.23294509151414308, "Islam religion": 0.23044096728307253, "Hindu religion": 0.21333333333333335, "Madurese": 0.21016949152542372, "Sport": 0.21621621621621623, "Indonesian language": 0.2559153175591532, "Physics": 0.2202020202020202, "Minangkabau culture": 0.25125628140703515, "Dayak language": 0.22935779816513763, "Sociology": 0.2056451612903226, "Economy": 0.22950819672131148, "Sundanese": 0.24632670700086431, "Javanese": 0.2721774193548387, "Civic education": 0.26180257510729615 } }, "prompt_3": { "accuracy": 0.23900126844248615, "category_acc": { "History": 0.20281124497991967, "Geography": 0.20612244897959184, "Lampungic": 0.2585034013605442, "Social science": 0.2570951585976628, "Balinese": 0.2653927813163482, "Makassarese": 0.1881720430107527, "Banjarese": 0.2222222222222222, "Chemistry": 0.1635036496350365, "Biology": 0.2165680473372781, "Science": 0.23839009287925697, "Christian religion": 0.26865671641791045, "Art": 0.24292845257903495, "Islam religion": 0.21621621621621623, "Hindu religion": 0.21333333333333335, "Madurese": 0.21694915254237288, "Sport": 0.21621621621621623, "Indonesian language": 0.2559153175591532, "Physics": 0.22424242424242424, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.3211009174311927, "Sociology": 0.2056451612903226, "Economy": 0.25, "Sundanese": 0.27052722558340536, "Javanese": 0.2540322580645161, "Civic education": 0.2632331902718169 } }, "prompt_4": { "accuracy": 0.24113759262968154, "category_acc": { "History": 0.19477911646586346, "Geography": 0.20816326530612245, "Lampungic": 0.2857142857142857, "Social science": 0.2654424040066778, "Balinese": 0.25690021231422505, "Makassarese": 0.20967741935483872, "Banjarese": 0.2222222222222222, "Chemistry": 0.1678832116788321, "Biology": 0.22248520710059172, "Science": 0.24251805985552116, "Christian religion": 0.29850746268656714, "Art": 0.23627287853577372, "Islam religion": 0.21763869132290184, "Hindu religion": 0.21333333333333335, "Madurese": 0.2135593220338983, "Sport": 0.1891891891891892, "Indonesian language": 0.2602739726027397, "Physics": 0.2383838383838384, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.29357798165137616, "Sociology": 0.20161290322580644, "Economy": 0.23975409836065573, "Sundanese": 0.26879861711322384, "Javanese": 0.25806451612903225, "Civic education": 0.2675250357653791 } }, "prompt_5": { "accuracy": 0.24240603511582884, "category_acc": { "History": 0.1927710843373494, "Geography": 0.21224489795918366, "Lampungic": 0.272108843537415, "Social science": 0.2637729549248748, "Balinese": 0.25477707006369427, "Makassarese": 0.1827956989247312, "Banjarese": 0.2152777777777778, "Chemistry": 0.17956204379562044, "Biology": 0.23076923076923078, "Science": 0.22910216718266255, "Christian religion": 0.27860696517412936, "Art": 0.24625623960066556, "Islam religion": 0.23897581792318634, "Hindu religion": 0.25333333333333335, "Madurese": 0.2033898305084746, "Sport": 0.18243243243243243, "Indonesian language": 0.2636986301369863, "Physics": 0.21414141414141413, "Minangkabau culture": 0.24623115577889448, "Dayak language": 0.27522935779816515, "Sociology": 0.20766129032258066, "Economy": 0.23770491803278687, "Sundanese": 0.2662057044079516, "Javanese": 0.2651209677419355, "Civic education": 0.2703862660944206 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.06407218520167664 }, "prompt_2": { "bleu_score": 0.06586220325650263 }, "prompt_3": { "bleu_score": 0.06381655662484863 }, "prompt_4": { "bleu_score": 0.06512354313124509 }, "prompt_5": { "bleu_score": 0.06038537914699977 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.03229032150259142 }, "prompt_2": { "bleu_score": 0.031199003066047656 }, "prompt_3": { "bleu_score": 0.030584311151675803 }, "prompt_4": { "bleu_score": 0.03385343853968681 }, "prompt_5": { "bleu_score": 0.031062141329664804 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.008319239259946272 }, "prompt_2": { "bleu_score": 0.012894931676973479 }, "prompt_3": { "bleu_score": 0.013587427733671357 }, "prompt_4": { "bleu_score": 0.012603283241102662 }, "prompt_5": { "bleu_score": 0.01183829943175568 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.057558239225713494 }, "prompt_2": { "bleu_score": 0.05813270509840964 }, "prompt_3": { "bleu_score": 0.05639021390265389 }, "prompt_4": { "bleu_score": 0.06088213042386755 }, "prompt_5": { "bleu_score": 0.056831945580458273 } }, "mmlu": { "prompt_1": { "accuracy": 0.34072345390898484 }, "prompt_2": { "accuracy": 0.3208868144690782 }, "prompt_3": { "accuracy": 0.3278879813302217 }, "prompt_4": { "accuracy": 0.3348891481913652 }, "prompt_5": { "accuracy": 0.3418903150525088 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.3511619592420451, "category_acc": { "high_school_european_history": 0.4878048780487805, "business_ethics": 0.3838383838383838, "clinical_knowledge": 0.3787878787878788, "medical_genetics": 0.45454545454545453, "high_school_us_history": 0.4236453201970443, "high_school_physics": 0.25333333333333335, "high_school_world_history": 0.4576271186440678, "virology": 0.36363636363636365, "high_school_microeconomics": 0.379746835443038, "econometrics": 0.24778761061946902, "college_computer_science": 0.2828282828282828, "high_school_biology": 0.32362459546925565, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.33807829181494664, "philosophy": 0.34838709677419355, "professional_medicine": 0.3025830258302583, "nutrition": 0.34098360655737703, "global_facts": 0.30303030303030304, "machine_learning": 0.23423423423423423, "security_studies": 0.36065573770491804, "public_relations": 0.41284403669724773, "professional_psychology": 0.353518821603928, "prehistory": 0.35294117647058826, "anatomy": 0.31343283582089554, "human_sexuality": 0.3769230769230769, "college_medicine": 0.3313953488372093, "high_school_government_and_politics": 0.4270833333333333, "college_chemistry": 0.20202020202020202, "logical_fallacies": 0.41975308641975306, "high_school_geography": 0.41116751269035534, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.3783783783783784, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.4485294117647059, "formal_logic": 0.344, "high_school_statistics": 0.26976744186046514, "international_law": 0.525, "high_school_mathematics": 0.2379182156133829, "high_school_computer_science": 0.31313131313131315, "conceptual_physics": 0.2863247863247863, "miscellaneous": 0.40664961636828645, "high_school_chemistry": 0.2623762376237624, "marketing": 0.6008583690987125, "professional_law": 0.3333333333333333, "management": 0.4411764705882353, "college_physics": 0.25742574257425743, "jurisprudence": 0.3925233644859813, "world_religions": 0.29411764705882354, "sociology": 0.535, "us_foreign_policy": 0.5151515151515151, "high_school_macroeconomics": 0.33161953727506427, "computer_security": 0.42424242424242425, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.39420289855072466, "electrical_engineering": 0.3194444444444444, "astronomy": 0.304635761589404, "college_biology": 0.2727272727272727 } }, "prompt_2": { "accuracy": 0.3492313192706471, "category_acc": { "high_school_european_history": 0.5, "business_ethics": 0.3434343434343434, "clinical_knowledge": 0.4166666666666667, "medical_genetics": 0.3939393939393939, "high_school_us_history": 0.4187192118226601, "high_school_physics": 0.3, "high_school_world_history": 0.4491525423728814, "virology": 0.3515151515151515, "high_school_microeconomics": 0.379746835443038, "econometrics": 0.30973451327433627, "college_computer_science": 0.2828282828282828, "high_school_biology": 0.2977346278317152, "abstract_algebra": 0.20202020202020202, "professional_accounting": 0.33807829181494664, "philosophy": 0.35161290322580646, "professional_medicine": 0.31365313653136534, "nutrition": 0.34098360655737703, "global_facts": 0.2222222222222222, "machine_learning": 0.3063063063063063, "security_studies": 0.38114754098360654, "public_relations": 0.3761467889908257, "professional_psychology": 0.36661211129296234, "prehistory": 0.34055727554179566, "anatomy": 0.30597014925373134, "human_sexuality": 0.36153846153846153, "college_medicine": 0.38372093023255816, "high_school_government_and_politics": 0.4322916666666667, "college_chemistry": 0.23232323232323232, "logical_fallacies": 0.41975308641975306, "high_school_geography": 0.39593908629441626, "elementary_mathematics": 0.27055702917771884, "human_aging": 0.3783783783783784, "college_mathematics": 0.24242424242424243, "high_school_psychology": 0.44485294117647056, "formal_logic": 0.272, "high_school_statistics": 0.2744186046511628, "international_law": 0.5166666666666667, "high_school_mathematics": 0.241635687732342, "high_school_computer_science": 0.3838383838383838, "conceptual_physics": 0.2777777777777778, "miscellaneous": 0.391304347826087, "high_school_chemistry": 0.2623762376237624, "marketing": 0.5708154506437768, "professional_law": 0.33659491193737767, "management": 0.46078431372549017, "college_physics": 0.2871287128712871, "jurisprudence": 0.37383177570093457, "world_religions": 0.29411764705882354, "sociology": 0.5, "us_foreign_policy": 0.46464646464646464, "high_school_macroeconomics": 0.3393316195372751, "computer_security": 0.3939393939393939, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.37681159420289856, "electrical_engineering": 0.2777777777777778, "astronomy": 0.32450331125827814, "college_biology": 0.2727272727272727 } }, "prompt_3": { "accuracy": 0.34000715051841257, "category_acc": { "high_school_european_history": 0.4817073170731707, "business_ethics": 0.35353535353535354, "clinical_knowledge": 0.38257575757575757, "medical_genetics": 0.37373737373737376, "high_school_us_history": 0.4039408866995074, "high_school_physics": 0.26, "high_school_world_history": 0.4533898305084746, "virology": 0.36363636363636365, "high_school_microeconomics": 0.350210970464135, "econometrics": 0.3185840707964602, "college_computer_science": 0.3434343434343434, "high_school_biology": 0.29449838187702265, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.3238434163701068, "philosophy": 0.34838709677419355, "professional_medicine": 0.2988929889298893, "nutrition": 0.36721311475409835, "global_facts": 0.26262626262626265, "machine_learning": 0.2882882882882883, "security_studies": 0.3524590163934426, "public_relations": 0.4036697247706422, "professional_psychology": 0.3502454991816694, "prehistory": 0.3281733746130031, "anatomy": 0.30597014925373134, "human_sexuality": 0.3153846153846154, "college_medicine": 0.3430232558139535, "high_school_government_and_politics": 0.4166666666666667, "college_chemistry": 0.21212121212121213, "logical_fallacies": 0.41358024691358025, "high_school_geography": 0.38071065989847713, "elementary_mathematics": 0.23076923076923078, "human_aging": 0.36036036036036034, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.4227941176470588, "formal_logic": 0.288, "high_school_statistics": 0.2558139534883721, "international_law": 0.5, "high_school_mathematics": 0.21933085501858737, "high_school_computer_science": 0.36363636363636365, "conceptual_physics": 0.25213675213675213, "miscellaneous": 0.38235294117647056, "high_school_chemistry": 0.2722772277227723, "marketing": 0.5407725321888412, "professional_law": 0.33529028049575993, "management": 0.4215686274509804, "college_physics": 0.27722772277227725, "jurisprudence": 0.3644859813084112, "world_religions": 0.2823529411764706, "sociology": 0.505, "us_foreign_policy": 0.4444444444444444, "high_school_macroeconomics": 0.3213367609254499, "computer_security": 0.35353535353535354, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.3710144927536232, "electrical_engineering": 0.2847222222222222, "astronomy": 0.33112582781456956, "college_biology": 0.3006993006993007 } }, "prompt_4": { "accuracy": 0.3471576689309975, "category_acc": { "high_school_european_history": 0.49390243902439024, "business_ethics": 0.3434343434343434, "clinical_knowledge": 0.38257575757575757, "medical_genetics": 0.40404040404040403, "high_school_us_history": 0.4236453201970443, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.4703389830508475, "virology": 0.3515151515151515, "high_school_microeconomics": 0.37130801687763715, "econometrics": 0.25663716814159293, "college_computer_science": 0.32323232323232326, "high_school_biology": 0.3042071197411003, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.33807829181494664, "philosophy": 0.34838709677419355, "professional_medicine": 0.3173431734317343, "nutrition": 0.3540983606557377, "global_facts": 0.21212121212121213, "machine_learning": 0.2702702702702703, "security_studies": 0.36475409836065575, "public_relations": 0.42201834862385323, "professional_psychology": 0.353518821603928, "prehistory": 0.33126934984520123, "anatomy": 0.27611940298507465, "human_sexuality": 0.35384615384615387, "college_medicine": 0.3430232558139535, "high_school_government_and_politics": 0.4114583333333333, "college_chemistry": 0.21212121212121213, "logical_fallacies": 0.4074074074074074, "high_school_geography": 0.4010152284263959, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.3918918918918919, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.43566176470588236, "formal_logic": 0.328, "high_school_statistics": 0.26976744186046514, "international_law": 0.5, "high_school_mathematics": 0.2527881040892193, "high_school_computer_science": 0.32323232323232326, "conceptual_physics": 0.3034188034188034, "miscellaneous": 0.3938618925831202, "high_school_chemistry": 0.27722772277227725, "marketing": 0.5793991416309013, "professional_law": 0.33268101761252444, "management": 0.45098039215686275, "college_physics": 0.26732673267326734, "jurisprudence": 0.3644859813084112, "world_religions": 0.27058823529411763, "sociology": 0.54, "us_foreign_policy": 0.47474747474747475, "high_school_macroeconomics": 0.3393316195372751, "computer_security": 0.41414141414141414, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.3565217391304348, "electrical_engineering": 0.3194444444444444, "astronomy": 0.32450331125827814, "college_biology": 0.27972027972027974 } }, "prompt_5": { "accuracy": 0.34522702895959956, "category_acc": { "high_school_european_history": 0.4817073170731707, "business_ethics": 0.37373737373737376, "clinical_knowledge": 0.35984848484848486, "medical_genetics": 0.3838383838383838, "high_school_us_history": 0.43349753694581283, "high_school_physics": 0.24, "high_school_world_history": 0.4703389830508475, "virology": 0.3333333333333333, "high_school_microeconomics": 0.33755274261603374, "econometrics": 0.22123893805309736, "college_computer_science": 0.31313131313131315, "high_school_biology": 0.3106796116504854, "abstract_algebra": 0.24242424242424243, "professional_accounting": 0.35231316725978645, "philosophy": 0.3419354838709677, "professional_medicine": 0.31365313653136534, "nutrition": 0.3442622950819672, "global_facts": 0.31313131313131315, "machine_learning": 0.25225225225225223, "security_studies": 0.36065573770491804, "public_relations": 0.3761467889908257, "professional_psychology": 0.35515548281505727, "prehistory": 0.34674922600619196, "anatomy": 0.29850746268656714, "human_sexuality": 0.38461538461538464, "college_medicine": 0.313953488372093, "high_school_government_and_politics": 0.4166666666666667, "college_chemistry": 0.21212121212121213, "logical_fallacies": 0.41975308641975306, "high_school_geography": 0.4010152284263959, "elementary_mathematics": 0.2625994694960212, "human_aging": 0.4009009009009009, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.42463235294117646, "formal_logic": 0.32, "high_school_statistics": 0.26976744186046514, "international_law": 0.5083333333333333, "high_school_mathematics": 0.2379182156133829, "high_school_computer_science": 0.3434343434343434, "conceptual_physics": 0.2863247863247863, "miscellaneous": 0.39514066496163686, "high_school_chemistry": 0.2623762376237624, "marketing": 0.592274678111588, "professional_law": 0.32811480756686234, "management": 0.4411764705882353, "college_physics": 0.25742574257425743, "jurisprudence": 0.3925233644859813, "world_religions": 0.27058823529411763, "sociology": 0.53, "us_foreign_policy": 0.46464646464646464, "high_school_macroeconomics": 0.34190231362467866, "computer_security": 0.40404040404040403, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.3710144927536232, "electrical_engineering": 0.2986111111111111, "astronomy": 0.31125827814569534, "college_biology": 0.2937062937062937 } } }, "c_eval": { "prompt_1": { "accuracy": 0.2436849925705795 }, "prompt_2": { "accuracy": 0.2436849925705795 }, "prompt_3": { "accuracy": 0.24442793462109955 }, "prompt_4": { "accuracy": 0.24145616641901932 }, "prompt_5": { "accuracy": 0.24294205052005943 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.26214196762141967, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.14285714285714285, "college_physics": 0.16666666666666666, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.21666666666666667, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.22448979591836735, "high_school_politics": 0.0, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.2962962962962963, "law": 0.3103448275862069, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.25, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.3076923076923077, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.3148148148148148, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3148148148148148, "physician": 0.2962962962962963 } }, "prompt_2": { "accuracy": 0.261519302615193, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.14285714285714285, "college_physics": 0.16666666666666666, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.21666666666666667, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.22448979591836735, "high_school_politics": 0.0, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.2962962962962963, "law": 0.3103448275862069, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.25, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.3076923076923077, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.3148148148148148, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3148148148148148, "physician": 0.2962962962962963 } }, "prompt_3": { "accuracy": 0.261519302615193, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.14285714285714285, "college_physics": 0.16666666666666666, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.21666666666666667, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.22448979591836735, "high_school_politics": 0.0, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.2962962962962963, "law": 0.3103448275862069, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.25, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.3076923076923077, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.3148148148148148, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3148148148148148, "physician": 0.2962962962962963 } }, "prompt_4": { "accuracy": 0.26525529265255293, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.14285714285714285, "college_physics": 0.20833333333333334, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.21666666666666667, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.22448979591836735, "high_school_politics": 0.0, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.3103448275862069, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.25, "high_school_history": 0.04, "middle_school_history": 0.3333333333333333, "civil_servant": 0.3076923076923077, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.3148148148148148, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.3148148148148148, "physician": 0.2962962962962963 } }, "prompt_5": { "accuracy": 0.2627646326276463, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.14285714285714285, "college_physics": 0.16666666666666666, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.21666666666666667, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.22448979591836735, "high_school_politics": 0.0, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.2962962962962963, "law": 0.3103448275862069, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.25, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.3076923076923077, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.3148148148148148, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3148148148148148, "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.21863799283154123 }, "prompt_2": { "accuracy": 0.2222222222222222 }, "prompt_3": { "accuracy": 0.2222222222222222 }, "prompt_4": { "accuracy": 0.25448028673835127 }, "prompt_5": { "accuracy": 0.21863799283154123 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2522880331548955, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.25, "ancient_chinese": 0.2682926829268293, "arts": 0.28125, "astronomy": 0.19393939393939394, "business_ethics": 0.24880382775119617, "chinese_civil_service_exam": 0.23125, "chinese_driving_rule": 0.2824427480916031, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.308411214953271, "chinese_history": 0.23529411764705882, "chinese_literature": 0.2549019607843137, "chinese_teacher_qualification": 0.30726256983240224, "clinical_knowledge": 0.27848101265822783, "college_actuarial_science": 0.24528301886792453, "college_education": 0.16822429906542055, "college_engineering_hydrology": 0.2358490566037736, "college_law": 0.25, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.2641509433962264, "college_medicine": 0.2454212454212454, "computer_science": 0.2107843137254902, "computer_security": 0.21637426900584794, "conceptual_physics": 0.25170068027210885, "construction_project_management": 0.22302158273381295, "economics": 0.27044025157232704, "education": 0.25766871165644173, "electrical_engineering": 0.23255813953488372, "elementary_chinese": 0.23015873015873015, "elementary_commonsense": 0.2828282828282828, "elementary_information_and_technology": 0.25630252100840334, "elementary_mathematics": 0.23478260869565218, "ethnology": 0.25925925925925924, "food_science": 0.24475524475524477, "genetics": 0.2784090909090909, "global_facts": 0.2751677852348993, "high_school_biology": 0.21301775147928995, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.3170731707317073, "high_school_physics": 0.20909090909090908, "high_school_politics": 0.3146853146853147, "human_sexuality": 0.2619047619047619, "international_law": 0.2702702702702703, "journalism": 0.22674418604651161, "jurisprudence": 0.23357664233576642, "legal_and_moral_basis": 0.2803738317757009, "logical": 0.2601626016260163, "machine_learning": 0.22950819672131148, "management": 0.2619047619047619, "marketing": 0.2611111111111111, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25, "nutrition": 0.25517241379310346, "philosophy": 0.22857142857142856, "professional_accounting": 0.28, "professional_law": 0.22748815165876776, "professional_medicine": 0.2473404255319149, "professional_psychology": 0.23275862068965517, "public_relations": 0.26436781609195403, "security_study": 0.26666666666666666, "sociology": 0.2831858407079646, "sports_science": 0.24848484848484848, "traditional_chinese_medicine": 0.23243243243243245, "virology": 0.25443786982248523, "world_history": 0.2422360248447205, "world_religions": 0.275 } }, "prompt_2": { "accuracy": 0.25125194266965983, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.24324324324324326, "ancient_chinese": 0.2804878048780488, "arts": 0.28125, "astronomy": 0.19393939393939394, "business_ethics": 0.24880382775119617, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.2824427480916031, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.308411214953271, "chinese_history": 0.23839009287925697, "chinese_literature": 0.24509803921568626, "chinese_teacher_qualification": 0.30726256983240224, "clinical_knowledge": 0.26582278481012656, "college_actuarial_science": 0.2169811320754717, "college_education": 0.1588785046728972, "college_engineering_hydrology": 0.2358490566037736, "college_law": 0.24074074074074073, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.27358490566037735, "college_medicine": 0.23076923076923078, "computer_science": 0.22058823529411764, "computer_security": 0.21637426900584794, "conceptual_physics": 0.2653061224489796, "construction_project_management": 0.2302158273381295, "economics": 0.27044025157232704, "education": 0.25766871165644173, "electrical_engineering": 0.23837209302325582, "elementary_chinese": 0.23412698412698413, "elementary_commonsense": 0.2828282828282828, "elementary_information_and_technology": 0.25210084033613445, "elementary_mathematics": 0.23043478260869565, "ethnology": 0.25925925925925924, "food_science": 0.23776223776223776, "genetics": 0.26704545454545453, "global_facts": 0.2751677852348993, "high_school_biology": 0.21301775147928995, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.31097560975609756, "high_school_physics": 0.2, "high_school_politics": 0.3146853146853147, "human_sexuality": 0.2619047619047619, "international_law": 0.2648648648648649, "journalism": 0.22674418604651161, "jurisprudence": 0.23357664233576642, "legal_and_moral_basis": 0.2803738317757009, "logical": 0.2601626016260163, "machine_learning": 0.21311475409836064, "management": 0.2619047619047619, "marketing": 0.2611111111111111, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.23275862068965517, "nutrition": 0.2482758620689655, "philosophy": 0.22857142857142856, "professional_accounting": 0.2857142857142857, "professional_law": 0.23696682464454977, "professional_medicine": 0.2473404255319149, "professional_psychology": 0.23275862068965517, "public_relations": 0.26436781609195403, "security_study": 0.26666666666666666, "sociology": 0.2831858407079646, "sports_science": 0.24848484848484848, "traditional_chinese_medicine": 0.22702702702702704, "virology": 0.26627218934911245, "world_history": 0.2484472049689441, "world_religions": 0.275 } }, "prompt_3": { "accuracy": 0.25116560179589015, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.24324324324324326, "ancient_chinese": 0.2804878048780488, "arts": 0.28125, "astronomy": 0.19393939393939394, "business_ethics": 0.24401913875598086, "chinese_civil_service_exam": 0.2375, "chinese_driving_rule": 0.2824427480916031, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.308411214953271, "chinese_history": 0.23529411764705882, "chinese_literature": 0.24509803921568626, "chinese_teacher_qualification": 0.30726256983240224, "clinical_knowledge": 0.270042194092827, "college_actuarial_science": 0.2641509433962264, "college_education": 0.1588785046728972, "college_engineering_hydrology": 0.2641509433962264, "college_law": 0.25, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.23076923076923078, "computer_science": 0.20588235294117646, "computer_security": 0.23391812865497075, "conceptual_physics": 0.2653061224489796, "construction_project_management": 0.2302158273381295, "economics": 0.27044025157232704, "education": 0.25766871165644173, "electrical_engineering": 0.23255813953488372, "elementary_chinese": 0.2261904761904762, "elementary_commonsense": 0.2828282828282828, "elementary_information_and_technology": 0.25630252100840334, "elementary_mathematics": 0.2391304347826087, "ethnology": 0.25925925925925924, "food_science": 0.24475524475524477, "genetics": 0.2727272727272727, "global_facts": 0.2751677852348993, "high_school_biology": 0.20710059171597633, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.2796610169491525, "high_school_mathematics": 0.3048780487804878, "high_school_physics": 0.19090909090909092, "high_school_politics": 0.32167832167832167, "human_sexuality": 0.2619047619047619, "international_law": 0.2702702702702703, "journalism": 0.22674418604651161, "jurisprudence": 0.23357664233576642, "legal_and_moral_basis": 0.2803738317757009, "logical": 0.2601626016260163, "machine_learning": 0.22131147540983606, "management": 0.2619047619047619, "marketing": 0.2611111111111111, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.23275862068965517, "nutrition": 0.25517241379310346, "philosophy": 0.23809523809523808, "professional_accounting": 0.2857142857142857, "professional_law": 0.23696682464454977, "professional_medicine": 0.2473404255319149, "professional_psychology": 0.23275862068965517, "public_relations": 0.26436781609195403, "security_study": 0.25925925925925924, "sociology": 0.2831858407079646, "sports_science": 0.24848484848484848, "traditional_chinese_medicine": 0.22702702702702704, "virology": 0.25443786982248523, "world_history": 0.2422360248447205, "world_religions": 0.2875 } }, "prompt_4": { "accuracy": 0.25211535140735625, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.23648648648648649, "ancient_chinese": 0.2682926829268293, "arts": 0.275, "astronomy": 0.22424242424242424, "business_ethics": 0.24880382775119617, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.33587786259541985, "chinese_food_culture": 0.27941176470588236, "chinese_foreign_policy": 0.2897196261682243, "chinese_history": 0.23219814241486067, "chinese_literature": 0.24509803921568626, "chinese_teacher_qualification": 0.2569832402234637, "clinical_knowledge": 0.2869198312236287, "college_actuarial_science": 0.2169811320754717, "college_education": 0.16822429906542055, "college_engineering_hydrology": 0.2641509433962264, "college_law": 0.25, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.29245283018867924, "college_medicine": 0.23809523809523808, "computer_science": 0.2107843137254902, "computer_security": 0.23391812865497075, "conceptual_physics": 0.2857142857142857, "construction_project_management": 0.2158273381294964, "economics": 0.27044025157232704, "education": 0.2085889570552147, "electrical_engineering": 0.23255813953488372, "elementary_chinese": 0.21428571428571427, "elementary_commonsense": 0.26262626262626265, "elementary_information_and_technology": 0.2689075630252101, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.3037037037037037, "food_science": 0.2867132867132867, "genetics": 0.26136363636363635, "global_facts": 0.28187919463087246, "high_school_biology": 0.23076923076923078, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.2457627118644068, "high_school_mathematics": 0.32926829268292684, "high_school_physics": 0.17272727272727273, "high_school_politics": 0.2867132867132867, "human_sexuality": 0.23015873015873015, "international_law": 0.2864864864864865, "journalism": 0.2441860465116279, "jurisprudence": 0.25060827250608275, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.2764227642276423, "machine_learning": 0.20491803278688525, "management": 0.21904761904761905, "marketing": 0.25555555555555554, "marxist_theory": 0.25396825396825395, "modern_chinese": 0.28448275862068967, "nutrition": 0.2896551724137931, "philosophy": 0.24761904761904763, "professional_accounting": 0.22285714285714286, "professional_law": 0.21800947867298578, "professional_medicine": 0.23138297872340424, "professional_psychology": 0.2629310344827586, "public_relations": 0.25862068965517243, "security_study": 0.26666666666666666, "sociology": 0.252212389380531, "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.26627218934911245, "world_history": 0.2670807453416149, "world_religions": 0.2875 } }, "prompt_5": { "accuracy": 0.2518563287860473, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.24324324324324326, "ancient_chinese": 0.27439024390243905, "arts": 0.275, "astronomy": 0.2, "business_ethics": 0.24401913875598086, "chinese_civil_service_exam": 0.2375, "chinese_driving_rule": 0.2824427480916031, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.3177570093457944, "chinese_history": 0.23529411764705882, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.30726256983240224, "clinical_knowledge": 0.270042194092827, "college_actuarial_science": 0.2358490566037736, "college_education": 0.1588785046728972, "college_engineering_hydrology": 0.2830188679245283, "college_law": 0.25, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.29245283018867924, "college_medicine": 0.22344322344322345, "computer_science": 0.21568627450980393, "computer_security": 0.22807017543859648, "conceptual_physics": 0.2585034013605442, "construction_project_management": 0.2302158273381295, "economics": 0.27044025157232704, "education": 0.25766871165644173, "electrical_engineering": 0.23837209302325582, "elementary_chinese": 0.23015873015873015, "elementary_commonsense": 0.2828282828282828, "elementary_information_and_technology": 0.2605042016806723, "elementary_mathematics": 0.22608695652173913, "ethnology": 0.25925925925925924, "food_science": 0.23776223776223776, "genetics": 0.2727272727272727, "global_facts": 0.2751677852348993, "high_school_biology": 0.22485207100591717, "high_school_chemistry": 0.22727272727272727, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.3170731707317073, "high_school_physics": 0.19090909090909092, "high_school_politics": 0.32167832167832167, "human_sexuality": 0.2619047619047619, "international_law": 0.2756756756756757, "journalism": 0.22674418604651161, "jurisprudence": 0.2384428223844282, "legal_and_moral_basis": 0.2803738317757009, "logical": 0.2601626016260163, "machine_learning": 0.21311475409836064, "management": 0.2619047619047619, "marketing": 0.2611111111111111, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.23275862068965517, "nutrition": 0.25517241379310346, "philosophy": 0.22857142857142856, "professional_accounting": 0.28, "professional_law": 0.23222748815165878, "professional_medicine": 0.2473404255319149, "professional_psychology": 0.23275862068965517, "public_relations": 0.26436781609195403, "security_study": 0.26666666666666666, "sociology": 0.2831858407079646, "sports_science": 0.24848484848484848, "traditional_chinese_medicine": 0.22702702702702704, "virology": 0.25443786982248523, "world_history": 0.2484472049689441, "world_religions": 0.275 } } }, "zbench": { "prompt_1": { "accuracy": 0.21212121212121213 }, "prompt_2": { "accuracy": 0.21212121212121213 }, "prompt_3": { "accuracy": 0.21212121212121213 }, "prompt_4": { "accuracy": 0.24242424242424243 }, "prompt_5": { "accuracy": 0.18181818181818182 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.15454545454545454 }, "prompt_2": { "accuracy": 0.15454545454545454 }, "prompt_3": { "accuracy": 0.13636363636363635 }, "prompt_4": { "accuracy": 0.22954545454545455 }, "prompt_5": { "accuracy": 0.29545454545454547 } }, "ocnli": { "prompt_1": { "accuracy": 0.3474576271186441 }, "prompt_2": { "accuracy": 0.33864406779661016 }, "prompt_3": { "accuracy": 0.33661016949152545 }, "prompt_4": { "accuracy": 0.348135593220339 }, "prompt_5": { "accuracy": 0.348135593220339 } }, "c3": { "prompt_1": { "accuracy": 0.27673896783844426 }, "prompt_2": { "accuracy": 0.28085265519820496 }, "prompt_3": { "accuracy": 0.27898279730740466 }, "prompt_4": { "accuracy": 0.27486910994764396 }, "prompt_5": { "accuracy": 0.27898279730740466 } }, "dream": { "prompt_1": { "accuracy": 0.7339539441450269 }, "prompt_2": { "accuracy": 0.732484076433121 }, "prompt_3": { "accuracy": 0.7354238118569328 }, "prompt_4": { "accuracy": 0.7275845173934345 }, "prompt_5": { "accuracy": 0.732484076433121 } }, "samsum": { "prompt_1": { "rouge1": 0.488484040273741, "rouge2": 0.23998086324118986, "rougeL": 0.4064468657172112, "avg_rouge": 0.37830392307738075 }, "prompt_2": { "rouge1": 0.4849018980243311, "rouge2": 0.237551200146336, "rougeL": 0.40312373253579026, "avg_rouge": 0.37519227690215246 }, "prompt_3": { "rouge1": 0.4895295318667229, "rouge2": 0.24117191448518707, "rougeL": 0.40644791748868025, "avg_rouge": 0.37904978794686334 }, "prompt_4": { "rouge1": 0.4866886581108752, "rouge2": 0.2394549990808094, "rougeL": 0.4058628019738131, "avg_rouge": 0.37733548638849923 }, "prompt_5": { "rouge1": 0.4883999304094335, "rouge2": 0.2396140799702983, "rougeL": 0.40482381144754065, "avg_rouge": 0.37761260727575746 } }, "dialogsum": { "prompt_1": { "rouge1": 0.2655584776294273, "rouge2": 0.08400121911084955, "rougeL": 0.2043071525241852, "avg_rouge": 0.184622283088154 }, "prompt_2": { "rouge1": 0.2739249487548124, "rouge2": 0.08831062539365067, "rougeL": 0.2093411832754166, "avg_rouge": 0.1905255858079599 }, "prompt_3": { "rouge1": 0.23397379438148797, "rouge2": 0.06834623109477767, "rougeL": 0.17495323413594485, "avg_rouge": 0.15909108653740348 }, "prompt_4": { "rouge1": 0.23679257285465277, "rouge2": 0.06983511641069447, "rougeL": 0.17733703138085358, "avg_rouge": 0.16132157354873358 }, "prompt_5": { "rouge1": 0.2708748715755265, "rouge2": 0.0875519573364796, "rougeL": 0.20647193742575837, "avg_rouge": 0.18829958877925482 } }, "sst2": { "prompt_1": { "accuracy": 0.9094036697247706 }, "prompt_2": { "accuracy": 0.908256880733945 }, "prompt_3": { "accuracy": 0.9036697247706422 }, "prompt_4": { "accuracy": 0.908256880733945 }, "prompt_5": { "accuracy": 0.9139908256880734 } }, "cola": { "prompt_1": { "accuracy": 0.5081495685522531 }, "prompt_2": { "accuracy": 0.6864813039309684 }, "prompt_3": { "accuracy": 0.660594439117929 }, "prompt_4": { "accuracy": 0.6078619367209971 }, "prompt_5": { "accuracy": 0.5225311601150527 } }, "qqp": { "prompt_1": { "accuracy": 0.81 }, "prompt_2": { "accuracy": 0.8175 }, "prompt_3": { "accuracy": 0.8055 }, "prompt_4": { "accuracy": 0.8175 }, "prompt_5": { "accuracy": 0.812 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.5 } }, "qnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } }, "rte": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "mrpc": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.6 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "flan_t5_large": { "model_size": "0.78B", "model_link": "https://huggingface.co/google/flan-t5-large", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.7676470588235293, "language_acc": { "Spanish": 0.9319327731092437, "English": 0.9705882352941176, "Chinese": 0.37563025210084033, "Vietnamese": 0.7924369747899159 }, "consistency_score_2": 0.6124649859943977, "consistency_score_3": 0.44621848739495795, "consistency_score_4": 0.32184873949579834, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9361344537815126, "Spanish,Chinese": 0.38487394957983195, "Spanish,Vietnamese": 0.7798319327731092, "English,Chinese": 0.3764705882352941, "English,Vietnamese": 0.788235294117647, "Chinese,Vietnamese": 0.4092436974789916 }, "3_combine": { "Spanish,English,Chinese": 0.3638655462184874, "Spanish,English,Vietnamese": 0.761344537815126, "Spanish,Chinese,Vietnamese": 0.3302521008403361, "English,Chinese,Vietnamese": 0.32941176470588235 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.32184873949579834 } }, "AC3_2": 0.6813315583503264, "AC3_3": 0.564376030739652, "AC3_4": 0.45354234246421293 }, "prompt_2": { "overall_acc": 0.7485294117647059, "language_acc": { "Spanish": 0.9168067226890756, "English": 0.9638655462184874, "Chinese": 0.35798319327731093, "Vietnamese": 0.7554621848739496 }, "consistency_score_2": 0.59593837535014, "consistency_score_3": 0.42626050420168066, "consistency_score_4": 0.3042016806722689, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9201680672268907, "Spanish,Chinese": 0.3739495798319328, "Spanish,Vietnamese": 0.7546218487394958, "English,Chinese": 0.3621848739495798, "English,Vietnamese": 0.7563025210084033, "Chinese,Vietnamese": 0.4084033613445378 }, "3_combine": { "Spanish,English,Chinese": 0.346218487394958, "Spanish,English,Vietnamese": 0.7277310924369748, "Spanish,Chinese,Vietnamese": 0.3176470588235294, "English,Chinese,Vietnamese": 0.3134453781512605 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3042016806722689 } }, "AC3_2": 0.6635746959366482, "AC3_3": 0.5431924808087655, "AC3_4": 0.4325965229025488 }, "prompt_3": { "overall_acc": 0.7565126050420168, "language_acc": { "Spanish": 0.9218487394957983, "English": 0.965546218487395, "Chinese": 0.36722689075630255, "Vietnamese": 0.7714285714285715 }, "consistency_score_2": 0.6046218487394958, "consistency_score_3": 0.4373949579831933, "consistency_score_4": 0.31512605042016806, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9277310924369748, "Spanish,Chinese": 0.3840336134453782, "Spanish,Vietnamese": 0.7638655462184873, "English,Chinese": 0.37058823529411766, "English,Vietnamese": 0.7714285714285715, "Chinese,Vietnamese": 0.41008403361344536 }, "3_combine": { "Spanish,English,Chinese": 0.3596638655462185, "Spanish,English,Vietnamese": 0.7453781512605042, "Spanish,Chinese,Vietnamese": 0.3235294117647059, "English,Chinese,Vietnamese": 0.32100840336134456 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.31512605042016806 } }, "AC3_2": 0.6720923837477901, "AC3_3": 0.5543055581789739, "AC3_4": 0.44492017543898615 }, "prompt_4": { "overall_acc": 0.7596638655462185, "language_acc": { "Spanish": 0.926890756302521, "English": 0.9680672268907563, "Chinese": 0.36050420168067226, "Vietnamese": 0.7831932773109244 }, "consistency_score_2": 0.6058823529411764, "consistency_score_3": 0.4373949579831933, "consistency_score_4": 0.3126050420168067, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9277310924369748, "Spanish,Chinese": 0.37478991596638656, "Spanish,Vietnamese": 0.7789915966386555, "English,Chinese": 0.36554621848739494, "English,Vietnamese": 0.7823529411764706, "Chinese,Vietnamese": 0.40588235294117647 }, "3_combine": { "Spanish,English,Chinese": 0.3521008403361345, "Spanish,English,Vietnamese": 0.7554621848739496, "Spanish,Chinese,Vietnamese": 0.3226890756302521, "English,Chinese,Vietnamese": 0.31932773109243695 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3126050420168067 } }, "AC3_2": 0.6741140270999553, "AC3_3": 0.5551492340944547, "AC3_4": 0.44293880557498183 }, "prompt_5": { "overall_acc": 0.7495798319327731, "language_acc": { "Spanish": 0.9159663865546218, "English": 0.9647058823529412, "Chinese": 0.3621848739495798, "Vietnamese": 0.7554621848739496 }, "consistency_score_2": 0.6012605042016806, "consistency_score_3": 0.4315126050420168, "consistency_score_4": 0.3100840336134454, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9176470588235294, "Spanish,Chinese": 0.38235294117647056, "Spanish,Vietnamese": 0.7554621848739496, "English,Chinese": 0.3680672268907563, "English,Vietnamese": 0.7554621848739496, "Chinese,Vietnamese": 0.42857142857142855 }, "3_combine": { "Spanish,English,Chinese": 0.35294117647058826, "Spanish,English,Vietnamese": 0.7285714285714285, "Spanish,Chinese,Vietnamese": 0.32521008403361346, "English,Chinese,Vietnamese": 0.31932773109243695 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3100840336134454 } }, "AC3_2": 0.667277598392779, "AC3_3": 0.5477185964637349, "AC3_4": 0.43869144798904225 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.31904761904761897, "language_acc": { "Filipino": 0.26666666666666666, "Vietnamese": 0.2866666666666667, "Chinese": 0.28, "Spanish": 0.34, "Malay": 0.24666666666666667, "Indonesian": 0.36, "English": 0.4533333333333333 }, "consistency_score_2": 0.5473015873015873, "consistency_score_3": 0.381904761904762, "consistency_score_4": 0.29599999999999993, "consistency_score_5": 0.24285714285714283, "consistency_score_6": 0.2066666666666667, "consistency_score_7": 0.18, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5333333333333333, "Filipino,Chinese": 0.56, "Filipino,Spanish": 0.54, "Filipino,Malay": 0.6266666666666667, "Filipino,Indonesian": 0.58, "Filipino,English": 0.5066666666666667, "Vietnamese,Chinese": 0.7, "Vietnamese,Spanish": 0.5466666666666666, "Vietnamese,Malay": 0.62, "Vietnamese,Indonesian": 0.58, "Vietnamese,English": 0.47333333333333333, "Chinese,Spanish": 0.5066666666666667, "Chinese,Malay": 0.6066666666666667, "Chinese,Indonesian": 0.5866666666666667, "Chinese,English": 0.44666666666666666, "Spanish,Malay": 0.4533333333333333, "Spanish,Indonesian": 0.5, "Spanish,English": 0.5533333333333333, "Malay,Indonesian": 0.6, "Malay,English": 0.48, "Indonesian,English": 0.49333333333333335 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.44, "Filipino,Vietnamese,Spanish": 0.36666666666666664, "Filipino,Vietnamese,Malay": 0.4533333333333333, "Filipino,Vietnamese,Indonesian": 0.3933333333333333, "Filipino,Vietnamese,English": 0.31333333333333335, "Filipino,Chinese,Spanish": 0.36666666666666664, "Filipino,Chinese,Malay": 0.46, "Filipino,Chinese,Indonesian": 0.4266666666666667, "Filipino,Chinese,English": 0.32666666666666666, "Filipino,Spanish,Malay": 0.4, "Filipino,Spanish,Indonesian": 0.36666666666666664, "Filipino,Spanish,English": 0.38, "Filipino,Malay,Indonesian": 0.46, "Filipino,Malay,English": 0.37333333333333335, "Filipino,Indonesian,English": 0.35333333333333333, "Vietnamese,Chinese,Spanish": 0.42, "Vietnamese,Chinese,Malay": 0.5066666666666667, "Vietnamese,Chinese,Indonesian": 0.47333333333333333, "Vietnamese,Chinese,English": 0.36, "Vietnamese,Spanish,Malay": 0.36, "Vietnamese,Spanish,Indonesian": 0.36666666666666664, "Vietnamese,Spanish,English": 0.3466666666666667, "Vietnamese,Malay,Indonesian": 0.43333333333333335, "Vietnamese,Malay,English": 0.3466666666666667, "Vietnamese,Indonesian,English": 0.3333333333333333, "Chinese,Spanish,Malay": 0.36, "Chinese,Spanish,Indonesian": 0.38, "Chinese,Spanish,English": 0.3333333333333333, "Chinese,Malay,Indonesian": 0.44666666666666666, "Chinese,Malay,English": 0.3333333333333333, "Chinese,Indonesian,English": 0.3466666666666667, "Spanish,Malay,Indonesian": 0.3333333333333333, "Spanish,Malay,English": 0.30666666666666664, "Spanish,Indonesian,English": 0.34, "Malay,Indonesian,English": 0.36 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Malay": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Chinese,English": 0.26, "Filipino,Vietnamese,Spanish,Malay": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.36, "Filipino,Vietnamese,Malay,English": 0.2866666666666667, "Filipino,Vietnamese,Indonesian,English": 0.26, "Filipino,Chinese,Spanish,Malay": 0.32666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Chinese,Spanish,English": 0.26, "Filipino,Chinese,Malay,Indonesian": 0.38, "Filipino,Chinese,Malay,English": 0.3, "Filipino,Chinese,Indonesian,English": 0.28, "Filipino,Spanish,Malay,Indonesian": 0.3, "Filipino,Spanish,Malay,English": 0.29333333333333333, "Filipino,Spanish,Indonesian,English": 0.28, "Filipino,Malay,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Malay": 0.32, "Vietnamese,Chinese,Spanish,Indonesian": 0.3333333333333333, "Vietnamese,Chinese,Spanish,English": 0.2733333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Chinese,Malay,English": 0.28, "Vietnamese,Chinese,Indonesian,English": 0.29333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,Malay,English": 0.24, "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Vietnamese,Malay,Indonesian,English": 0.28, "Chinese,Spanish,Malay,Indonesian": 0.3, "Chinese,Spanish,Malay,English": 0.25333333333333335, "Chinese,Spanish,Indonesian,English": 0.28, "Chinese,Malay,Indonesian,English": 0.28, "Spanish,Malay,Indonesian,English": 0.24 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.32, "Filipino,Vietnamese,Chinese,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, "Filipino,Vietnamese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.24, "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.24, "Vietnamese,Spanish,Malay,Indonesian,English": 0.2, "Chinese,Spanish,Malay,Indonesian,English": 0.22 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.2, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 } }, "AC3_2": 0.4031059693474576, "AC3_3": 0.3476578674452261, "AC3_4": 0.3070919788909187, "AC3_5": 0.27578692489038653, "AC3_6": 0.25084541058030413, "AC3_7": 0.2301526717096068 }, "prompt_2": { "overall_acc": 0.31428571428571433, "language_acc": { "Filipino": 0.30666666666666664, "Vietnamese": 0.28, "Chinese": 0.26, "Spanish": 0.3466666666666667, "Malay": 0.23333333333333334, "Indonesian": 0.30666666666666664, "English": 0.4666666666666667 }, "consistency_score_2": 0.5374603174603175, "consistency_score_3": 0.3678095238095238, "consistency_score_4": 0.2811428571428571, "consistency_score_5": 0.22825396825396824, "consistency_score_6": 0.1904761904761905, "consistency_score_7": 0.16, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.47333333333333333, "Filipino,Chinese": 0.5333333333333333, "Filipino,Spanish": 0.5333333333333333, "Filipino,Malay": 0.6133333333333333, "Filipino,Indonesian": 0.6066666666666667, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.6266666666666667, "Vietnamese,Spanish": 0.5133333333333333, "Vietnamese,Malay": 0.5933333333333334, "Vietnamese,Indonesian": 0.54, "Vietnamese,English": 0.47333333333333333, "Chinese,Spanish": 0.49333333333333335, "Chinese,Malay": 0.6266666666666667, "Chinese,Indonesian": 0.5666666666666667, "Chinese,English": 0.46, "Spanish,Malay": 0.4533333333333333, "Spanish,Indonesian": 0.48, "Spanish,English": 0.6, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.46, "Indonesian,English": 0.5 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.38, "Filipino,Vietnamese,Spanish": 0.3333333333333333, "Filipino,Vietnamese,Malay": 0.4, "Filipino,Vietnamese,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,English": 0.30666666666666664, "Filipino,Chinese,Spanish": 0.35333333333333333, "Filipino,Chinese,Malay": 0.44666666666666666, "Filipino,Chinese,Indonesian": 0.4266666666666667, "Filipino,Chinese,English": 0.32666666666666666, "Filipino,Spanish,Malay": 0.38, "Filipino,Spanish,Indonesian": 0.38666666666666666, "Filipino,Spanish,English": 0.38, "Filipino,Malay,Indonesian": 0.49333333333333335, "Filipino,Malay,English": 0.35333333333333333, "Filipino,Indonesian,English": 0.38, "Vietnamese,Chinese,Spanish": 0.36666666666666664, "Vietnamese,Chinese,Malay": 0.4666666666666667, "Vietnamese,Chinese,Indonesian": 0.4, "Vietnamese,Chinese,English": 0.3333333333333333, "Vietnamese,Spanish,Malay": 0.34, "Vietnamese,Spanish,Indonesian": 0.32, "Vietnamese,Spanish,English": 0.3333333333333333, "Vietnamese,Malay,Indonesian": 0.44, "Vietnamese,Malay,English": 0.32666666666666666, "Vietnamese,Indonesian,English": 0.32, "Chinese,Spanish,Malay": 0.35333333333333333, "Chinese,Spanish,Indonesian": 0.34, "Chinese,Spanish,English": 0.34, "Chinese,Malay,Indonesian": 0.4666666666666667, "Chinese,Malay,English": 0.3333333333333333, "Chinese,Indonesian,English": 0.3333333333333333, "Spanish,Malay,Indonesian": 0.34, "Spanish,Malay,English": 0.31333333333333335, "Spanish,Indonesian,English": 0.3333333333333333, "Malay,Indonesian,English": 0.36 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Chinese,English": 0.26, "Filipino,Vietnamese,Spanish,Malay": 0.28, "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Malay,English": 0.2733333333333333, "Filipino,Vietnamese,Indonesian,English": 0.26, "Filipino,Chinese,Spanish,Malay": 0.30666666666666664, "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Chinese,Spanish,English": 0.26, "Filipino,Chinese,Malay,Indonesian": 0.38666666666666666, "Filipino,Chinese,Malay,English": 0.2866666666666667, "Filipino,Chinese,Indonesian,English": 0.29333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, "Filipino,Spanish,Malay,English": 0.28, "Filipino,Spanish,Indonesian,English": 0.29333333333333333, "Filipino,Malay,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Malay": 0.28, "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Spanish,English": 0.25333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.36, "Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.26, "Vietnamese,Spanish,Malay,Indonesian": 0.26, "Vietnamese,Spanish,Malay,English": 0.24, "Vietnamese,Spanish,Indonesian,English": 0.22, "Vietnamese,Malay,Indonesian,English": 0.2733333333333333, "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, "Chinese,Spanish,Malay,English": 0.24666666666666667, "Chinese,Spanish,Indonesian,English": 0.24666666666666667, "Chinese,Malay,Indonesian,English": 0.29333333333333333, "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, "Filipino,Vietnamese,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.2, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.24, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Chinese,Spanish,Malay,Indonesian,English": 0.22 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 } }, "AC3_2": 0.3966348969237751, "AC3_3": 0.33894762032769654, "AC3_4": 0.29679188369021525, "AC3_5": 0.2644487168282948, "AC3_6": 0.2371967654516605, "AC3_7": 0.2120481927263754 }, "prompt_3": { "overall_acc": 0.30666666666666664, "language_acc": { "Filipino": 0.29333333333333333, "Vietnamese": 0.24666666666666667, "Chinese": 0.2733333333333333, "Spanish": 0.34, "Malay": 0.24666666666666667, "Indonesian": 0.30666666666666664, "English": 0.44 }, "consistency_score_2": 0.567936507936508, "consistency_score_3": 0.4022857142857142, "consistency_score_4": 0.3154285714285714, "consistency_score_5": 0.26412698412698415, "consistency_score_6": 0.23047619047619047, "consistency_score_7": 0.20666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5533333333333333, "Filipino,Chinese": 0.5933333333333334, "Filipino,Spanish": 0.5533333333333333, "Filipino,Malay": 0.6733333333333333, "Filipino,Indonesian": 0.62, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.6666666666666666, "Vietnamese,Spanish": 0.5533333333333333, "Vietnamese,Malay": 0.6066666666666667, "Vietnamese,Indonesian": 0.5866666666666667, "Vietnamese,English": 0.48, "Chinese,Spanish": 0.5333333333333333, "Chinese,Malay": 0.6466666666666666, "Chinese,Indonesian": 0.6066666666666667, "Chinese,English": 0.48, "Spanish,Malay": 0.5, "Spanish,Indonesian": 0.5066666666666667, "Spanish,English": 0.58, "Malay,Indonesian": 0.7066666666666667, "Malay,English": 0.5066666666666667, "Indonesian,English": 0.48 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.46, "Filipino,Vietnamese,Spanish": 0.38666666666666666, "Filipino,Vietnamese,Malay": 0.4666666666666667, "Filipino,Vietnamese,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,English": 0.32666666666666666, "Filipino,Chinese,Spanish": 0.4, "Filipino,Chinese,Malay": 0.49333333333333335, "Filipino,Chinese,Indonesian": 0.4666666666666667, "Filipino,Chinese,English": 0.34, "Filipino,Spanish,Malay": 0.41333333333333333, "Filipino,Spanish,Indonesian": 0.38666666666666666, "Filipino,Spanish,English": 0.37333333333333335, "Filipino,Malay,Indonesian": 0.54, "Filipino,Malay,English": 0.38, "Filipino,Indonesian,English": 0.36666666666666664, "Vietnamese,Chinese,Spanish": 0.4266666666666667, "Vietnamese,Chinese,Malay": 0.5, "Vietnamese,Chinese,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,English": 0.36, "Vietnamese,Spanish,Malay": 0.38, "Vietnamese,Spanish,Indonesian": 0.36, "Vietnamese,Spanish,English": 0.3466666666666667, "Vietnamese,Malay,Indonesian": 0.48, "Vietnamese,Malay,English": 0.36, "Vietnamese,Indonesian,English": 0.32666666666666666, "Chinese,Spanish,Malay": 0.4, "Chinese,Spanish,Indonesian": 0.38666666666666666, "Chinese,Spanish,English": 0.36, "Chinese,Malay,Indonesian": 0.5066666666666667, "Chinese,Malay,English": 0.36666666666666664, "Chinese,Indonesian,English": 0.3466666666666667, "Spanish,Malay,Indonesian": 0.38666666666666666, "Spanish,Malay,English": 0.35333333333333333, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.3933333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.4, "Filipino,Vietnamese,Chinese,Indonesian": 0.38, "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.3333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Spanish,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian": 0.3933333333333333, "Filipino,Vietnamese,Malay,English": 0.30666666666666664, "Filipino,Vietnamese,Indonesian,English": 0.28, "Filipino,Chinese,Spanish,Malay": 0.3466666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Chinese,Spanish,English": 0.2733333333333333, "Filipino,Chinese,Malay,Indonesian": 0.4266666666666667, "Filipino,Chinese,Malay,English": 0.30666666666666664, "Filipino,Chinese,Indonesian,English": 0.3, "Filipino,Spanish,Malay,Indonesian": 0.3333333333333333, "Filipino,Spanish,Malay,English": 0.29333333333333333, "Filipino,Spanish,Indonesian,English": 0.28, "Filipino,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.34, "Vietnamese,Chinese,Spanish,Indonesian": 0.3333333333333333, "Vietnamese,Chinese,Spanish,English": 0.28, "Vietnamese,Chinese,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Chinese,Malay,English": 0.30666666666666664, "Vietnamese,Chinese,Indonesian,English": 0.28, "Vietnamese,Spanish,Malay,Indonesian": 0.3, "Vietnamese,Spanish,Malay,English": 0.26666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.24, "Vietnamese,Malay,Indonesian,English": 0.3, "Chinese,Spanish,Malay,Indonesian": 0.3333333333333333, "Chinese,Spanish,Malay,English": 0.28, "Chinese,Spanish,Indonesian,English": 0.28, "Chinese,Malay,Indonesian,English": 0.32, "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.3, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.24, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.3, "Filipino,Chinese,Spanish,Malay,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.24, "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.24, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.2733333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.26 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 } }, "AC3_2": 0.39827707194478257, "AC3_3": 0.34802794191759406, "AC3_4": 0.3109859154429676, "AC3_5": 0.2838116425160916, "AC3_6": 0.2631678486507696, "AC3_7": 0.24692640687830436 }, "prompt_4": { "overall_acc": 0.3152380952380952, "language_acc": { "Filipino": 0.29333333333333333, "Vietnamese": 0.28, "Chinese": 0.2733333333333333, "Spanish": 0.34, "Malay": 0.24, "Indonesian": 0.32666666666666666, "English": 0.4533333333333333 }, "consistency_score_2": 0.5666666666666667, "consistency_score_3": 0.4081904761904762, "consistency_score_4": 0.3259047619047619, "consistency_score_5": 0.27650793650793654, "consistency_score_6": 0.24380952380952384, "consistency_score_7": 0.22, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5266666666666666, "Filipino,Chinese": 0.6, "Filipino,Spanish": 0.5866666666666667, "Filipino,Malay": 0.68, "Filipino,Indonesian": 0.62, "Filipino,English": 0.5466666666666666, "Vietnamese,Chinese": 0.6466666666666666, "Vietnamese,Spanish": 0.5533333333333333, "Vietnamese,Malay": 0.5666666666666667, "Vietnamese,Indonesian": 0.5333333333333333, "Vietnamese,English": 0.47333333333333333, "Chinese,Spanish": 0.52, "Chinese,Malay": 0.64, "Chinese,Indonesian": 0.6, "Chinese,English": 0.48, "Spanish,Malay": 0.5066666666666667, "Spanish,Indonesian": 0.5733333333333334, "Spanish,English": 0.5866666666666667, "Malay,Indonesian": 0.66, "Malay,English": 0.4866666666666667, "Indonesian,English": 0.5133333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.44, "Filipino,Vietnamese,Spanish": 0.4066666666666667, "Filipino,Vietnamese,Malay": 0.44, "Filipino,Vietnamese,Indonesian": 0.4, "Filipino,Vietnamese,English": 0.34, "Filipino,Chinese,Spanish": 0.42, "Filipino,Chinese,Malay": 0.5, "Filipino,Chinese,Indonesian": 0.46, "Filipino,Chinese,English": 0.38, "Filipino,Spanish,Malay": 0.44, "Filipino,Spanish,Indonesian": 0.44, "Filipino,Spanish,English": 0.44, "Filipino,Malay,Indonesian": 0.52, "Filipino,Malay,English": 0.41333333333333333, "Filipino,Indonesian,English": 0.4066666666666667, "Vietnamese,Chinese,Spanish": 0.4, "Vietnamese,Chinese,Malay": 0.48, "Vietnamese,Chinese,Indonesian": 0.4266666666666667, "Vietnamese,Chinese,English": 0.3466666666666667, "Vietnamese,Spanish,Malay": 0.36666666666666664, "Vietnamese,Spanish,Indonesian": 0.36666666666666664, "Vietnamese,Spanish,English": 0.37333333333333335, "Vietnamese,Malay,Indonesian": 0.44, "Vietnamese,Malay,English": 0.3333333333333333, "Vietnamese,Indonesian,English": 0.32, "Chinese,Spanish,Malay": 0.38666666666666666, "Chinese,Spanish,Indonesian": 0.4066666666666667, "Chinese,Spanish,English": 0.3933333333333333, "Chinese,Malay,Indonesian": 0.49333333333333335, "Chinese,Malay,English": 0.36666666666666664, "Chinese,Indonesian,English": 0.38, "Spanish,Malay,Indonesian": 0.4066666666666667, "Spanish,Malay,English": 0.36, "Spanish,Indonesian,English": 0.38666666666666666, "Malay,Indonesian,English": 0.4066666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.34, "Filipino,Vietnamese,Chinese,Malay": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Chinese,English": 0.29333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.3466666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Spanish,English": 0.3, "Filipino,Vietnamese,Malay,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Malay,English": 0.3, "Filipino,Vietnamese,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay": 0.35333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.36, "Filipino,Chinese,Spanish,English": 0.32666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.42, "Filipino,Chinese,Malay,English": 0.32666666666666666, "Filipino,Chinese,Indonesian,English": 0.3333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.36666666666666664, "Filipino,Spanish,Malay,English": 0.3333333333333333, "Filipino,Spanish,Indonesian,English": 0.3466666666666667, "Filipino,Malay,Indonesian,English": 0.35333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.32, "Vietnamese,Chinese,Spanish,Indonesian": 0.30666666666666664, "Vietnamese,Chinese,Spanish,English": 0.29333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Chinese,Malay,English": 0.2866666666666667, "Vietnamese,Chinese,Indonesian,English": 0.28, "Vietnamese,Spanish,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Spanish,Malay,English": 0.26666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian,English": 0.3, "Chinese,Spanish,Malay,Indonesian": 0.3466666666666667, "Chinese,Spanish,Malay,English": 0.29333333333333333, "Chinese,Spanish,Indonesian,English": 0.32, "Chinese,Malay,Indonesian,English": 0.3333333333333333, "Spanish,Malay,Indonesian,English": 0.30666666666666664 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.26, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.26666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.26, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32, "Filipino,Chinese,Spanish,Malay,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.29333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.3, "Filipino,Spanish,Malay,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, "Vietnamese,Chinese,Spanish,Malay,English": 0.24, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 } }, "AC3_2": 0.4051115910267782, "AC3_3": 0.3557426213806507, "AC3_4": 0.32048269807977076, "AC3_5": 0.29460555891199636, "AC3_6": 0.274960655422545, "AC3_7": 0.2591459074248927 }, "prompt_5": { "overall_acc": 0.31523809523809526, "language_acc": { "Filipino": 0.30666666666666664, "Vietnamese": 0.25333333333333335, "Chinese": 0.26, "Spanish": 0.3466666666666667, "Malay": 0.26, "Indonesian": 0.28, "English": 0.5 }, "consistency_score_2": 0.5269841269841271, "consistency_score_3": 0.3487619047619048, "consistency_score_4": 0.25904761904761914, "consistency_score_5": 0.206984126984127, "consistency_score_6": 0.1742857142857143, "consistency_score_7": 0.15333333333333332, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5, "Filipino,Chinese": 0.54, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.6066666666666667, "Filipino,Indonesian": 0.5866666666666667, "Filipino,English": 0.47333333333333333, "Vietnamese,Chinese": 0.6266666666666667, "Vietnamese,Spanish": 0.48, "Vietnamese,Malay": 0.6066666666666667, "Vietnamese,Indonesian": 0.5066666666666667, "Vietnamese,English": 0.4533333333333333, "Chinese,Spanish": 0.4666666666666667, "Chinese,Malay": 0.6333333333333333, "Chinese,Indonesian": 0.5466666666666666, "Chinese,English": 0.46, "Spanish,Malay": 0.42, "Spanish,Indonesian": 0.48, "Spanish,English": 0.56, "Malay,Indonesian": 0.6733333333333333, "Malay,English": 0.4866666666666667, "Indonesian,English": 0.48 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.3933333333333333, "Filipino,Vietnamese,Spanish": 0.30666666666666664, "Filipino,Vietnamese,Malay": 0.4, "Filipino,Vietnamese,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,English": 0.3, "Filipino,Chinese,Spanish": 0.32, "Filipino,Chinese,Malay": 0.43333333333333335, "Filipino,Chinese,Indonesian": 0.3933333333333333, "Filipino,Chinese,English": 0.30666666666666664, "Filipino,Spanish,Malay": 0.3333333333333333, "Filipino,Spanish,Indonesian": 0.34, "Filipino,Spanish,English": 0.3333333333333333, "Filipino,Malay,Indonesian": 0.4666666666666667, "Filipino,Malay,English": 0.34, "Filipino,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish": 0.3466666666666667, "Vietnamese,Chinese,Malay": 0.48, "Vietnamese,Chinese,Indonesian": 0.38, "Vietnamese,Chinese,English": 0.32666666666666666, "Vietnamese,Spanish,Malay": 0.31333333333333335, "Vietnamese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,English": 0.30666666666666664, "Vietnamese,Malay,Indonesian": 0.4266666666666667, "Vietnamese,Malay,English": 0.34, "Vietnamese,Indonesian,English": 0.29333333333333333, "Chinese,Spanish,Malay": 0.3333333333333333, "Chinese,Spanish,Indonesian": 0.30666666666666664, "Chinese,Spanish,English": 0.32666666666666666, "Chinese,Malay,Indonesian": 0.46, "Chinese,Malay,English": 0.3466666666666667, "Chinese,Indonesian,English": 0.31333333333333335, "Spanish,Malay,Indonesian": 0.31333333333333335, "Spanish,Malay,English": 0.3, "Spanish,Indonesian,English": 0.29333333333333333, "Malay,Indonesian,English": 0.36666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Chinese,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.24, "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Malay,English": 0.26, "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay": 0.2733333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.26, "Filipino,Chinese,Spanish,English": 0.24, "Filipino,Chinese,Malay,Indonesian": 0.3466666666666667, "Filipino,Chinese,Malay,English": 0.26666666666666666, "Filipino,Chinese,Indonesian,English": 0.24666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Spanish,Malay,English": 0.24666666666666667, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay": 0.28, "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,Spanish,English": 0.24, "Vietnamese,Chinese,Malay,Indonesian": 0.34, "Vietnamese,Chinese,Malay,English": 0.28, "Vietnamese,Chinese,Indonesian,English": 0.24, "Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Spanish,Malay,English": 0.22, "Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Malay,Indonesian,English": 0.26, "Chinese,Spanish,Malay,Indonesian": 0.26, "Chinese,Spanish,Malay,English": 0.24, "Chinese,Spanish,Indonesian,English": 0.22, "Chinese,Malay,Indonesian,English": 0.2733333333333333, "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, "Filipino,Vietnamese,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, "Vietnamese,Chinese,Spanish,Malay,English": 0.2, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, "Vietnamese,Chinese,Malay,Indonesian,English": 0.22, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 } }, "AC3_2": 0.3944932062066381, "AC3_3": 0.3311537305186601, "AC3_4": 0.28439390344885984, "AC3_5": 0.24989086693724047, "AC3_6": 0.2244691494816607, "AC3_7": 0.2063143630996009 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3003246753246754, "language_acc": { "Indonesian": 0.26136363636363635, "English": 0.5113636363636364, "Filipino": 0.2556818181818182, "Spanish": 0.32954545454545453, "Chinese": 0.24431818181818182, "Malay": 0.24431818181818182, "Vietnamese": 0.2556818181818182 }, "consistency_score_2": 0.449404761904762, "consistency_score_3": 0.2490259740259741, "consistency_score_4": 0.14853896103896105, "consistency_score_5": 0.09063852813852814, "consistency_score_6": 0.05519480519480519, "consistency_score_7": 0.03409090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3352272727272727, "Indonesian,Filipino": 0.5909090909090909, "Indonesian,Spanish": 0.4659090909090909, "Indonesian,Chinese": 0.48295454545454547, "Indonesian,Malay": 0.6193181818181818, "Indonesian,Vietnamese": 0.5056818181818182, "English,Filipino": 0.3181818181818182, "English,Spanish": 0.4602272727272727, "English,Chinese": 0.2840909090909091, "English,Malay": 0.32386363636363635, "English,Vietnamese": 0.3409090909090909, "Filipino,Spanish": 0.4147727272727273, "Filipino,Chinese": 0.5056818181818182, "Filipino,Malay": 0.5511363636363636, "Filipino,Vietnamese": 0.5170454545454546, "Spanish,Chinese": 0.3522727272727273, "Spanish,Malay": 0.42613636363636365, "Spanish,Vietnamese": 0.4090909090909091, "Chinese,Malay": 0.44886363636363635, "Chinese,Vietnamese": 0.5738636363636364, "Malay,Vietnamese": 0.5113636363636364 }, "3_combine": { "Indonesian,English,Filipino": 0.19318181818181818, "Indonesian,English,Spanish": 0.22727272727272727, "Indonesian,English,Chinese": 0.1590909090909091, "Indonesian,English,Malay": 0.23863636363636365, "Indonesian,English,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Spanish": 0.30113636363636365, "Indonesian,Filipino,Chinese": 0.3522727272727273, "Indonesian,Filipino,Malay": 0.4431818181818182, "Indonesian,Filipino,Vietnamese": 0.35795454545454547, "Indonesian,Spanish,Chinese": 0.2215909090909091, "Indonesian,Spanish,Malay": 0.3125, "Indonesian,Spanish,Vietnamese": 0.25, "Indonesian,Chinese,Malay": 0.3409090909090909, "Indonesian,Chinese,Vietnamese": 0.3465909090909091, "Indonesian,Malay,Vietnamese": 0.36363636363636365, "English,Filipino,Spanish": 0.1875, "English,Filipino,Chinese": 0.14204545454545456, "English,Filipino,Malay": 0.17045454545454544, "English,Filipino,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese": 0.14772727272727273, "English,Spanish,Malay": 0.19886363636363635, "English,Spanish,Vietnamese": 0.19886363636363635, "English,Chinese,Malay": 0.14772727272727273, "English,Chinese,Vietnamese": 0.17045454545454544, "English,Malay,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Chinese": 0.2215909090909091, "Filipino,Spanish,Malay": 0.2784090909090909, "Filipino,Spanish,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay": 0.3181818181818182, "Filipino,Chinese,Vietnamese": 0.3522727272727273, "Filipino,Malay,Vietnamese": 0.32386363636363635, "Spanish,Chinese,Malay": 0.19886363636363635, "Spanish,Chinese,Vietnamese": 0.25, "Spanish,Malay,Vietnamese": 0.2215909090909091, "Chinese,Malay,Vietnamese": 0.3181818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.13068181818181818, "Indonesian,English,Filipino,Chinese": 0.10795454545454546, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese": 0.09090909090909091, "Indonesian,English,Spanish,Malay": 0.1590909090909091, "Indonesian,English,Spanish,Vietnamese": 0.10795454545454546, "Indonesian,English,Chinese,Malay": 0.125, "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Malay": 0.23863636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay": 0.26136363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese": 0.07954545454545454, "English,Filipino,Spanish,Malay": 0.11363636363636363, "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "English,Filipino,Chinese,Malay": 0.09090909090909091, "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, "English,Filipino,Malay,Vietnamese": 0.09659090909090909, "English,Spanish,Chinese,Malay": 0.07954545454545454, "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, "English,Spanish,Malay,Vietnamese": 0.11363636363636363, "English,Chinese,Malay,Vietnamese": 0.10795454545454546, "Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.23863636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Spanish,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, "English,Filipino,Chinese,Malay,Vietnamese": 0.0625, "English,Spanish,Chinese,Malay,Vietnamese": 0.0625, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 } }, "AC3_2": 0.3600427900741609, "AC3_3": 0.27228017249656605, "AC3_4": 0.19876822991743748, "AC3_5": 0.13925088748516634, "AC3_6": 0.09325149733485677, "AC3_7": 0.06123124446536174 }, "prompt_2": { "overall_acc": 0.30113636363636365, "language_acc": { "Indonesian": 0.24431818181818182, "English": 0.5113636363636364, "Filipino": 0.26704545454545453, "Spanish": 0.3181818181818182, "Chinese": 0.24431818181818182, "Malay": 0.23295454545454544, "Vietnamese": 0.2897727272727273 }, "consistency_score_2": 0.4396645021645022, "consistency_score_3": 0.2446428571428572, "consistency_score_4": 0.15194805194805192, "consistency_score_5": 0.0992965367965368, "consistency_score_6": 0.06655844155844155, "consistency_score_7": 0.045454545454545456, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3181818181818182, "Indonesian,Filipino": 0.5227272727272727, "Indonesian,Spanish": 0.4375, "Indonesian,Chinese": 0.48295454545454547, "Indonesian,Malay": 0.625, "Indonesian,Vietnamese": 0.48863636363636365, "English,Filipino": 0.3181818181818182, "English,Spanish": 0.4943181818181818, "English,Chinese": 0.2556818181818182, "English,Malay": 0.2727272727272727, "English,Vietnamese": 0.35795454545454547, "Filipino,Spanish": 0.4090909090909091, "Filipino,Chinese": 0.4431818181818182, "Filipino,Malay": 0.5625, "Filipino,Vietnamese": 0.4772727272727273, "Spanish,Chinese": 0.3693181818181818, "Spanish,Malay": 0.45454545454545453, "Spanish,Vietnamese": 0.4431818181818182, "Chinese,Malay": 0.4318181818181818, "Chinese,Vietnamese": 0.5568181818181818, "Malay,Vietnamese": 0.5113636363636364 }, "3_combine": { "Indonesian,English,Filipino": 0.17613636363636365, "Indonesian,English,Spanish": 0.2215909090909091, "Indonesian,English,Chinese": 0.1534090909090909, "Indonesian,English,Malay": 0.19886363636363635, "Indonesian,English,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Spanish": 0.2784090909090909, "Indonesian,Filipino,Chinese": 0.2897727272727273, "Indonesian,Filipino,Malay": 0.4147727272727273, "Indonesian,Filipino,Vietnamese": 0.3068181818181818, "Indonesian,Spanish,Chinese": 0.2215909090909091, "Indonesian,Spanish,Malay": 0.3352272727272727, "Indonesian,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,Chinese,Malay": 0.32386363636363635, "Indonesian,Chinese,Vietnamese": 0.3181818181818182, "Indonesian,Malay,Vietnamese": 0.36363636363636365, "English,Filipino,Spanish": 0.19318181818181818, "English,Filipino,Chinese": 0.13068181818181818, "English,Filipino,Malay": 0.1590909090909091, "English,Filipino,Vietnamese": 0.1875, "English,Spanish,Chinese": 0.14772727272727273, "English,Spanish,Malay": 0.19886363636363635, "English,Spanish,Vietnamese": 0.24431818181818182, "English,Chinese,Malay": 0.125, "English,Chinese,Vietnamese": 0.17045454545454544, "English,Malay,Vietnamese": 0.19886363636363635, "Filipino,Spanish,Chinese": 0.2159090909090909, "Filipino,Spanish,Malay": 0.2784090909090909, "Filipino,Spanish,Vietnamese": 0.25, "Filipino,Chinese,Malay": 0.2840909090909091, "Filipino,Chinese,Vietnamese": 0.29545454545454547, "Filipino,Malay,Vietnamese": 0.32954545454545453, "Spanish,Chinese,Malay": 0.22727272727272727, "Spanish,Chinese,Vietnamese": 0.26136363636363635, "Spanish,Malay,Vietnamese": 0.2840909090909091, "Chinese,Malay,Vietnamese": 0.3068181818181818 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.13636363636363635, "Indonesian,English,Filipino,Chinese": 0.10227272727272728, "Indonesian,English,Filipino,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese": 0.10795454545454546, "Indonesian,English,Spanish,Malay": 0.1590909090909091, "Indonesian,English,Spanish,Vietnamese": 0.1534090909090909, "Indonesian,English,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.1875, "Indonesian,Filipino,Chinese,Malay": 0.20454545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay": 0.1875, "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Chinese,Malay,Vietnamese": 0.23295454545454544, "English,Filipino,Spanish,Chinese": 0.09090909090909091, "English,Filipino,Spanish,Malay": 0.11931818181818182, "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, "English,Filipino,Chinese,Malay": 0.07954545454545454, "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, "English,Filipino,Malay,Vietnamese": 0.125, "English,Spanish,Chinese,Malay": 0.09090909090909091, "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Spanish,Malay,Vietnamese": 0.14204545454545456, "English,Chinese,Malay,Vietnamese": 0.10795454545454546, "Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.07386363636363637, "Indonesian,English,Filipino,Spanish,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.125, "Indonesian,English,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.125, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 } }, "AC3_2": 0.357448203683736, "AC3_3": 0.26996579412067256, "AC3_4": 0.20198039147361257, "AC3_5": 0.14934735868506224, "AC3_6": 0.10902067024929864, "AC3_7": 0.07898658716051893 }, "prompt_3": { "overall_acc": 0.288961038961039, "language_acc": { "Indonesian": 0.25, "English": 0.44886363636363635, "Filipino": 0.26136363636363635, "Spanish": 0.32386363636363635, "Chinese": 0.25, "Malay": 0.23295454545454544, "Vietnamese": 0.2556818181818182 }, "consistency_score_2": 0.4507575757575758, "consistency_score_3": 0.2568181818181819, "consistency_score_4": 0.16103896103896104, "consistency_score_5": 0.10470779220779221, "consistency_score_6": 0.0689935064935065, "consistency_score_7": 0.045454545454545456, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3465909090909091, "Indonesian,Filipino": 0.5625, "Indonesian,Spanish": 0.44886363636363635, "Indonesian,Chinese": 0.48295454545454547, "Indonesian,Malay": 0.6647727272727273, "Indonesian,Vietnamese": 0.4772727272727273, "English,Filipino": 0.29545454545454547, "English,Spanish": 0.48863636363636365, "English,Chinese": 0.30113636363636365, "English,Malay": 0.2840909090909091, "English,Vietnamese": 0.3352272727272727, "Filipino,Spanish": 0.45454545454545453, "Filipino,Chinese": 0.4772727272727273, "Filipino,Malay": 0.5681818181818182, "Filipino,Vietnamese": 0.4659090909090909, "Spanish,Chinese": 0.35795454545454547, "Spanish,Malay": 0.48295454545454547, "Spanish,Vietnamese": 0.4090909090909091, "Chinese,Malay": 0.4715909090909091, "Chinese,Vietnamese": 0.5681818181818182, "Malay,Vietnamese": 0.5227272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.19318181818181818, "Indonesian,English,Spanish": 0.22727272727272727, "Indonesian,English,Chinese": 0.18181818181818182, "Indonesian,English,Malay": 0.22727272727272727, "Indonesian,English,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish": 0.3068181818181818, "Indonesian,Filipino,Chinese": 0.32386363636363635, "Indonesian,Filipino,Malay": 0.44886363636363635, "Indonesian,Filipino,Vietnamese": 0.3125, "Indonesian,Spanish,Chinese": 0.21022727272727273, "Indonesian,Spanish,Malay": 0.35795454545454547, "Indonesian,Spanish,Vietnamese": 0.25, "Indonesian,Chinese,Malay": 0.35795454545454547, "Indonesian,Chinese,Vietnamese": 0.3181818181818182, "Indonesian,Malay,Vietnamese": 0.375, "English,Filipino,Spanish": 0.2159090909090909, "English,Filipino,Chinese": 0.1534090909090909, "English,Filipino,Malay": 0.17613636363636365, "English,Filipino,Vietnamese": 0.16477272727272727, "English,Spanish,Chinese": 0.17045454545454544, "English,Spanish,Malay": 0.2215909090909091, "English,Spanish,Vietnamese": 0.2215909090909091, "English,Chinese,Malay": 0.14772727272727273, "English,Chinese,Vietnamese": 0.19318181818181818, "English,Malay,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Chinese": 0.23863636363636365, "Filipino,Spanish,Malay": 0.32386363636363635, "Filipino,Spanish,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay": 0.3181818181818182, "Filipino,Chinese,Vietnamese": 0.3068181818181818, "Filipino,Malay,Vietnamese": 0.32386363636363635, "Spanish,Chinese,Malay": 0.23863636363636365, "Spanish,Chinese,Vietnamese": 0.2556818181818182, "Spanish,Malay,Vietnamese": 0.2727272727272727, "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.13636363636363635, "Indonesian,English,Filipino,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese": 0.10227272727272728, "Indonesian,English,Spanish,Malay": 0.17613636363636365, "Indonesian,English,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,English,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Malay": 0.2840909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Chinese,Malay": 0.26136363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Spanish,Chinese,Malay": 0.1875, "Indonesian,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Spanish,Chinese": 0.10227272727272728, "English,Filipino,Spanish,Malay": 0.14204545454545456, "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, "English,Filipino,Chinese,Malay": 0.09659090909090909, "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.10227272727272728, "English,Spanish,Chinese,Vietnamese": 0.13068181818181818, "English,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Chinese,Malay,Vietnamese": 0.11363636363636363, "Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.1875, "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.06818181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.125, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.06818181818181818, "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0625, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 } }, "AC3_2": 0.352164660456482, "AC3_3": 0.27194310740785554, "AC3_4": 0.20681771326526033, "AC3_5": 0.15371535676909345, "AC3_6": 0.11139087666588042, "AC3_7": 0.07855251542222984 }, "prompt_4": { "overall_acc": 0.2938311688311689, "language_acc": { "Indonesian": 0.22727272727272727, "English": 0.4943181818181818, "Filipino": 0.2556818181818182, "Spanish": 0.32954545454545453, "Chinese": 0.26136363636363635, "Malay": 0.23295454545454544, "Vietnamese": 0.2556818181818182 }, "consistency_score_2": 0.43290043290043295, "consistency_score_3": 0.23538961038961043, "consistency_score_4": 0.14090909090909093, "consistency_score_5": 0.08793290043290043, "consistency_score_6": 0.056818181818181816, "consistency_score_7": 0.03977272727272727, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.30113636363636365, "Indonesian,Filipino": 0.5681818181818182, "Indonesian,Spanish": 0.3977272727272727, "Indonesian,Chinese": 0.45454545454545453, "Indonesian,Malay": 0.6193181818181818, "Indonesian,Vietnamese": 0.48863636363636365, "English,Filipino": 0.3352272727272727, "English,Spanish": 0.4659090909090909, "English,Chinese": 0.30113636363636365, "English,Malay": 0.3068181818181818, "English,Vietnamese": 0.36363636363636365, "Filipino,Spanish": 0.4147727272727273, "Filipino,Chinese": 0.4715909090909091, "Filipino,Malay": 0.5625, "Filipino,Vietnamese": 0.48295454545454547, "Spanish,Chinese": 0.32386363636363635, "Spanish,Malay": 0.4318181818181818, "Spanish,Vietnamese": 0.3806818181818182, "Chinese,Malay": 0.42613636363636365, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.19318181818181818, "Indonesian,English,Spanish": 0.18181818181818182, "Indonesian,English,Chinese": 0.14772727272727273, "Indonesian,English,Malay": 0.20454545454545456, "Indonesian,English,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Spanish": 0.2727272727272727, "Indonesian,Filipino,Chinese": 0.3125, "Indonesian,Filipino,Malay": 0.4318181818181818, "Indonesian,Filipino,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Chinese": 0.18181818181818182, "Indonesian,Spanish,Malay": 0.2784090909090909, "Indonesian,Spanish,Vietnamese": 0.22727272727272727, "Indonesian,Chinese,Malay": 0.30113636363636365, "Indonesian,Chinese,Vietnamese": 0.3125, "Indonesian,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish": 0.19886363636363635, "English,Filipino,Chinese": 0.1590909090909091, "English,Filipino,Malay": 0.17613636363636365, "English,Filipino,Vietnamese": 0.1875, "English,Spanish,Chinese": 0.13636363636363635, "English,Spanish,Malay": 0.19318181818181818, "English,Spanish,Vietnamese": 0.19886363636363635, "English,Chinese,Malay": 0.14772727272727273, "English,Chinese,Vietnamese": 0.1875, "English,Malay,Vietnamese": 0.19886363636363635, "Filipino,Spanish,Chinese": 0.20454545454545456, "Filipino,Spanish,Malay": 0.2840909090909091, "Filipino,Spanish,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay": 0.2897727272727273, "Filipino,Chinese,Vietnamese": 0.29545454545454547, "Filipino,Malay,Vietnamese": 0.3181818181818182, "Spanish,Chinese,Malay": 0.19318181818181818, "Spanish,Chinese,Vietnamese": 0.20454545454545456, "Spanish,Malay,Vietnamese": 0.23295454545454544, "Chinese,Malay,Vietnamese": 0.2840909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.11931818181818182, "Indonesian,English,Filipino,Chinese": 0.10795454545454546, "Indonesian,English,Filipino,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese": 0.07386363636363637, "Indonesian,English,Spanish,Malay": 0.14204545454545456, "Indonesian,English,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Malay": 0.2159090909090909, "Indonesian,Filipino,Spanish,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Chinese,Malay": 0.23295454545454544, "Indonesian,Filipino,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Spanish,Chinese,Malay": 0.13636363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish,Chinese": 0.08522727272727272, "English,Filipino,Spanish,Malay": 0.11931818181818182, "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, "English,Filipino,Chinese,Malay": 0.09090909090909091, "English,Filipino,Chinese,Vietnamese": 0.10795454545454546, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.07954545454545454, "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, "English,Spanish,Malay,Vietnamese": 0.11931818181818182, "English,Chinese,Malay,Vietnamese": 0.11931818181818182, "Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Malay": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Chinese,Malay": 0.07386363636363637, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.0625, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 } }, "AC3_2": 0.3500594713813695, "AC3_3": 0.26138355504582134, "AC3_4": 0.1904745739259062, "AC3_5": 0.13535808624396273, "AC3_6": 0.09522306394590777, "AC3_7": 0.0700619331796178 }, "prompt_5": { "overall_acc": 0.3060064935064935, "language_acc": { "Indonesian": 0.2215909090909091, "English": 0.5340909090909091, "Filipino": 0.2556818181818182, "Spanish": 0.3352272727272727, "Chinese": 0.26136363636363635, "Malay": 0.24431818181818182, "Vietnamese": 0.2897727272727273 }, "consistency_score_2": 0.4293831168831169, "consistency_score_3": 0.23311688311688308, "consistency_score_4": 0.14139610389610388, "consistency_score_5": 0.08982683982683981, "consistency_score_6": 0.05844155844155845, "consistency_score_7": 0.03977272727272727, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3409090909090909, "Indonesian,Filipino": 0.5, "Indonesian,Spanish": 0.4090909090909091, "Indonesian,Chinese": 0.4431818181818182, "Indonesian,Malay": 0.6647727272727273, "Indonesian,Vietnamese": 0.4715909090909091, "English,Filipino": 0.32954545454545453, "English,Spanish": 0.4772727272727273, "English,Chinese": 0.2840909090909091, "English,Malay": 0.2840909090909091, "English,Vietnamese": 0.3806818181818182, "Filipino,Spanish": 0.38636363636363635, "Filipino,Chinese": 0.4375, "Filipino,Malay": 0.5454545454545454, "Filipino,Vietnamese": 0.45454545454545453, "Spanish,Chinese": 0.3125, "Spanish,Malay": 0.4318181818181818, "Spanish,Vietnamese": 0.4147727272727273, "Chinese,Malay": 0.4147727272727273, "Chinese,Vietnamese": 0.5625, "Malay,Vietnamese": 0.4715909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.19318181818181818, "Indonesian,English,Spanish": 0.21022727272727273, "Indonesian,English,Chinese": 0.1590909090909091, "Indonesian,English,Malay": 0.23295454545454544, "Indonesian,English,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Spanish": 0.22727272727272727, "Indonesian,Filipino,Chinese": 0.26704545454545453, "Indonesian,Filipino,Malay": 0.4090909090909091, "Indonesian,Filipino,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Chinese": 0.16477272727272727, "Indonesian,Spanish,Malay": 0.3125, "Indonesian,Spanish,Vietnamese": 0.23863636363636365, "Indonesian,Chinese,Malay": 0.30113636363636365, "Indonesian,Chinese,Vietnamese": 0.3068181818181818, "Indonesian,Malay,Vietnamese": 0.35795454545454547, "English,Filipino,Spanish": 0.19318181818181818, "English,Filipino,Chinese": 0.14772727272727273, "English,Filipino,Malay": 0.17045454545454544, "English,Filipino,Vietnamese": 0.19318181818181818, "English,Spanish,Chinese": 0.14204545454545456, "English,Spanish,Malay": 0.20454545454545456, "English,Spanish,Vietnamese": 0.23295454545454544, "English,Chinese,Malay": 0.13636363636363635, "English,Chinese,Vietnamese": 0.19318181818181818, "English,Malay,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Chinese": 0.1875, "Filipino,Spanish,Malay": 0.26136363636363635, "Filipino,Spanish,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay": 0.2784090909090909, "Filipino,Chinese,Vietnamese": 0.30113636363636365, "Filipino,Malay,Vietnamese": 0.2897727272727273, "Spanish,Chinese,Malay": 0.17613636363636365, "Spanish,Chinese,Vietnamese": 0.23295454545454544, "Spanish,Malay,Vietnamese": 0.23863636363636365, "Chinese,Malay,Vietnamese": 0.29545454545454547 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.125, "Indonesian,English,Filipino,Chinese": 0.10795454545454546, "Indonesian,English,Filipino,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese": 0.08522727272727272, "Indonesian,English,Spanish,Malay": 0.16477272727272727, "Indonesian,English,Spanish,Vietnamese": 0.14772727272727273, "Indonesian,English,Chinese,Malay": 0.11931818181818182, "Indonesian,English,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Chinese,Malay": 0.21022727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Spanish,Chinese,Malay": 0.125, "Indonesian,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,Spanish,Malay,Vietnamese": 0.1875, "Indonesian,Chinese,Malay,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish,Chinese": 0.09090909090909091, "English,Filipino,Spanish,Malay": 0.11931818181818182, "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, "English,Filipino,Chinese,Malay": 0.09090909090909091, "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.07954545454545454, "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Chinese,Malay,Vietnamese": 0.11363636363636363, "Filipino,Spanish,Chinese,Malay": 0.125, "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, "Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.0625, "Indonesian,English,Filipino,Spanish,Malay": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Spanish,Chinese,Malay": 0.0625, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,English,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 } }, "AC3_2": 0.3573453312205796, "AC3_3": 0.26463434182102813, "AC3_4": 0.19341919868257093, "AC3_5": 0.13888469694940525, "AC3_6": 0.098140167155555, "AC3_7": 0.07039585998817745 } }, "sg_eval": { "prompt_1": { "accuracy": 0.4368932038834951 }, "prompt_2": { "accuracy": 0.4368932038834951 }, "prompt_3": { "accuracy": 0.44660194174757284 }, "prompt_4": { "accuracy": 0.44660194174757284 }, "prompt_5": { "accuracy": 0.42718446601941745 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2857142857142857 }, "prompt_2": { "accuracy": 0.21904761904761905 }, "prompt_3": { "accuracy": 0.22857142857142856 }, "prompt_4": { "accuracy": 0.21904761904761905 }, "prompt_5": { "accuracy": 0.24761904761904763 } }, "us_eval": { "prompt_1": { "accuracy": 0.38317757009345793 }, "prompt_2": { "accuracy": 0.3644859813084112 }, "prompt_3": { "accuracy": 0.38317757009345793 }, "prompt_4": { "accuracy": 0.40186915887850466 }, "prompt_5": { "accuracy": 0.40186915887850466 } }, "ph_eval": { "prompt_1": { "accuracy": 0.24, "category_acc": { "brand": 0.1, "demographics": 0.0, "biology": 0.3, "history": 0.13333333333333333, "literature": 0.5, "politics": 0.4, "culture": 0.1, "film": 0.1, "law": 0.3, "geography": 0.4 } }, "prompt_2": { "accuracy": 0.23, "category_acc": { "brand": 0.1, "demographics": 0.2, "biology": 0.3, "history": 0.13333333333333333, "literature": 0.4, "politics": 0.3, "culture": 0.1, "film": 0.1, "law": 0.3, "geography": 0.4 } }, "prompt_3": { "accuracy": 0.23, "category_acc": { "brand": 0.1, "demographics": 0.2, "biology": 0.3, "history": 0.2, "literature": 0.3, "politics": 0.2, "culture": 0.1, "film": 0.1, "law": 0.3, "geography": 0.5 } }, "prompt_4": { "accuracy": 0.26, "category_acc": { "brand": 0.1, "demographics": 0.4, "biology": 0.3, "history": 0.2, "literature": 0.4, "politics": 0.4, "culture": 0.1, "film": 0.1, "law": 0.3, "geography": 0.4 } }, "prompt_5": { "accuracy": 0.25, "category_acc": { "brand": 0.1, "demographics": 0.2, "biology": 0.3, "history": 0.2, "literature": 0.3, "politics": 0.2, "culture": 0.1, "film": 0.1, "law": 0.3, "geography": 0.7 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.17858137998030377 }, "prompt_2": { "bleu_score": 0.17980302230654807 }, "prompt_3": { "bleu_score": 0.18216557256738 }, "prompt_4": { "bleu_score": 0.18530270379921296 }, "prompt_5": { "bleu_score": 0.16536410084591763 } }, "indommlu": { "prompt_1": { "accuracy": 0.2544896187996529, "category_acc": { "History": 0.25502008032128515, "Geography": 0.22448979591836735, "Lampungic": 0.2857142857142857, "Social science": 0.28046744574290483, "Balinese": 0.29936305732484075, "Makassarese": 0.2849462365591398, "Banjarese": 0.2916666666666667, "Chemistry": 0.20583941605839415, "Biology": 0.22248520710059172, "Science": 0.24664602683178535, "Christian religion": 0.30845771144278605, "Art": 0.259567387687188, "Islam religion": 0.2233285917496444, "Hindu religion": 0.2733333333333333, "Madurese": 0.2847457627118644, "Sport": 0.32432432432432434, "Indonesian language": 0.2693026151930262, "Physics": 0.21616161616161617, "Minangkabau culture": 0.25125628140703515, "Dayak language": 0.25688073394495414, "Sociology": 0.20766129032258066, "Economy": 0.22745901639344263, "Sundanese": 0.28608470181503887, "Javanese": 0.2540322580645161, "Civic education": 0.2374821173104435 } }, "prompt_2": { "accuracy": 0.25769410508044593, "category_acc": { "History": 0.25301204819277107, "Geography": 0.2163265306122449, "Lampungic": 0.29931972789115646, "Social science": 0.28213689482470783, "Balinese": 0.29936305732484075, "Makassarese": 0.2956989247311828, "Banjarese": 0.2916666666666667, "Chemistry": 0.21751824817518248, "Biology": 0.24142011834319527, "Science": 0.239422084623323, "Christian religion": 0.30845771144278605, "Art": 0.26622296173044924, "Islam religion": 0.24893314366998578, "Hindu religion": 0.24666666666666667, "Madurese": 0.27796610169491526, "Sport": 0.31756756756756754, "Indonesian language": 0.27179327521793273, "Physics": 0.20404040404040405, "Minangkabau culture": 0.271356783919598, "Dayak language": 0.26605504587155965, "Sociology": 0.19959677419354838, "Economy": 0.2069672131147541, "Sundanese": 0.29559204840103714, "Javanese": 0.2550403225806452, "Civic education": 0.2532188841201717 } }, "prompt_3": { "accuracy": 0.25936310835169235, "category_acc": { "History": 0.25903614457831325, "Geography": 0.22448979591836735, "Lampungic": 0.30612244897959184, "Social science": 0.2921535893155259, "Balinese": 0.3248407643312102, "Makassarese": 0.27956989247311825, "Banjarese": 0.2847222222222222, "Chemistry": 0.20437956204379562, "Biology": 0.23668639053254437, "Science": 0.239422084623323, "Christian religion": 0.3482587064676617, "Art": 0.26455906821963393, "Islam religion": 0.24324324324324326, "Hindu religion": 0.26666666666666666, "Madurese": 0.26101694915254237, "Sport": 0.30405405405405406, "Indonesian language": 0.26805728518057287, "Physics": 0.21414141414141413, "Minangkabau culture": 0.2663316582914573, "Dayak language": 0.23853211009174313, "Sociology": 0.20362903225806453, "Economy": 0.23155737704918034, "Sundanese": 0.3016421780466724, "Javanese": 0.2600806451612903, "Civic education": 0.25608011444921314 } }, "prompt_4": { "accuracy": 0.25475665932305225, "category_acc": { "History": 0.26104417670682734, "Geography": 0.21224489795918366, "Lampungic": 0.2925170068027211, "Social science": 0.2854757929883139, "Balinese": 0.3184713375796178, "Makassarese": 0.2849462365591398, "Banjarese": 0.2569444444444444, "Chemistry": 0.21751824817518248, "Biology": 0.20946745562130178, "Science": 0.24251805985552116, "Christian religion": 0.2885572139303483, "Art": 0.26788685524126454, "Islam religion": 0.23470839260312945, "Hindu religion": 0.26666666666666666, "Madurese": 0.2745762711864407, "Sport": 0.32432432432432434, "Indonesian language": 0.26525529265255293, "Physics": 0.2202020202020202, "Minangkabau culture": 0.2914572864321608, "Dayak language": 0.25688073394495414, "Sociology": 0.2056451612903226, "Economy": 0.1987704918032787, "Sundanese": 0.2990492653414002, "Javanese": 0.25806451612903225, "Civic education": 0.2374821173104435 } }, "prompt_5": { "accuracy": 0.2565591828559984, "category_acc": { "History": 0.2389558232931727, "Geography": 0.22040816326530613, "Lampungic": 0.3129251700680272, "Social science": 0.2570951585976628, "Balinese": 0.3184713375796178, "Makassarese": 0.2849462365591398, "Banjarese": 0.2847222222222222, "Chemistry": 0.19708029197080293, "Biology": 0.21301775147928995, "Science": 0.2435500515995872, "Christian religion": 0.2736318407960199, "Art": 0.2562396006655574, "Islam religion": 0.25177809388335703, "Hindu religion": 0.26, "Madurese": 0.2677966101694915, "Sport": 0.34459459459459457, "Indonesian language": 0.27677459526774595, "Physics": 0.21414141414141413, "Minangkabau culture": 0.271356783919598, "Dayak language": 0.24770642201834864, "Sociology": 0.20766129032258066, "Economy": 0.19672131147540983, "Sundanese": 0.3025064822817632, "Javanese": 0.2590725806451613, "Civic education": 0.2632331902718169 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.12760103596967726 }, "prompt_2": { "bleu_score": 0.12728182784238143 }, "prompt_3": { "bleu_score": 0.12596753154582943 }, "prompt_4": { "bleu_score": 0.12849571351474165 }, "prompt_5": { "bleu_score": 0.11764504442840339 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.03668322829140508 }, "prompt_2": { "bleu_score": 0.03663401506892185 }, "prompt_3": { "bleu_score": 0.037271563791032004 }, "prompt_4": { "bleu_score": 0.03656009783069245 }, "prompt_5": { "bleu_score": 0.03556398570352625 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.014429676623784958 }, "prompt_2": { "bleu_score": 0.016031539247195952 }, "prompt_3": { "bleu_score": 0.015074674723810412 }, "prompt_4": { "bleu_score": 0.01475034003355935 }, "prompt_5": { "bleu_score": 0.012008477296306986 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.1184246883322347 }, "prompt_2": { "bleu_score": 0.11819809679873207 }, "prompt_3": { "bleu_score": 0.11660955144396412 }, "prompt_4": { "bleu_score": 0.11840549537817129 }, "prompt_5": { "bleu_score": 0.11277536636039184 } }, "mmlu": { "prompt_1": { "accuracy": 0.456242707117853 }, "prompt_2": { "accuracy": 0.43990665110851807 }, "prompt_3": { "accuracy": 0.4364060676779463 }, "prompt_4": { "accuracy": 0.4364060676779463 }, "prompt_5": { "accuracy": 0.43990665110851807 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.41344297461565965, "category_acc": { "high_school_european_history": 0.5426829268292683, "business_ethics": 0.5555555555555556, "clinical_knowledge": 0.44696969696969696, "medical_genetics": 0.43434343434343436, "high_school_us_history": 0.5369458128078818, "high_school_physics": 0.31333333333333335, "high_school_world_history": 0.576271186440678, "virology": 0.34545454545454546, "high_school_microeconomics": 0.459915611814346, "econometrics": 0.25663716814159293, "college_computer_science": 0.30303030303030304, "high_school_biology": 0.42394822006472493, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.3416370106761566, "philosophy": 0.4161290322580645, "professional_medicine": 0.4095940959409594, "nutrition": 0.46557377049180326, "global_facts": 0.30303030303030304, "machine_learning": 0.24324324324324326, "security_studies": 0.5573770491803278, "public_relations": 0.47706422018348627, "professional_psychology": 0.3698854337152209, "prehistory": 0.4055727554179567, "anatomy": 0.3880597014925373, "human_sexuality": 0.4, "college_medicine": 0.4186046511627907, "high_school_government_and_politics": 0.546875, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.5246913580246914, "high_school_geography": 0.5482233502538071, "elementary_mathematics": 0.27851458885941643, "human_aging": 0.45495495495495497, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.5661764705882353, "formal_logic": 0.36, "high_school_statistics": 0.33488372093023255, "international_law": 0.5583333333333333, "high_school_mathematics": 0.2825278810408922, "high_school_computer_science": 0.36363636363636365, "conceptual_physics": 0.358974358974359, "miscellaneous": 0.5127877237851662, "high_school_chemistry": 0.29207920792079206, "marketing": 0.7167381974248928, "professional_law": 0.3378995433789954, "management": 0.6372549019607843, "college_physics": 0.31683168316831684, "jurisprudence": 0.4766355140186916, "world_religions": 0.38235294117647056, "sociology": 0.565, "us_foreign_policy": 0.5757575757575758, "high_school_macroeconomics": 0.43958868894601544, "computer_security": 0.47474747474747475, "moral_scenarios": 0.23937360178970918, "moral_disputes": 0.48985507246376814, "electrical_engineering": 0.4027777777777778, "astronomy": 0.37748344370860926, "college_biology": 0.3706293706293706 } }, "prompt_2": { "accuracy": 0.41329996424740795, "category_acc": { "high_school_european_history": 0.573170731707317, "business_ethics": 0.5050505050505051, "clinical_knowledge": 0.4431818181818182, "medical_genetics": 0.4444444444444444, "high_school_us_history": 0.5467980295566502, "high_school_physics": 0.30666666666666664, "high_school_world_history": 0.5889830508474576, "virology": 0.36363636363636365, "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.24778761061946902, "college_computer_science": 0.31313131313131315, "high_school_biology": 0.43042071197411, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.37722419928825623, "philosophy": 0.4161290322580645, "professional_medicine": 0.39114391143911437, "nutrition": 0.4557377049180328, "global_facts": 0.29292929292929293, "machine_learning": 0.3153153153153153, "security_studies": 0.5573770491803278, "public_relations": 0.46788990825688076, "professional_psychology": 0.37479541734860883, "prehistory": 0.3931888544891641, "anatomy": 0.417910447761194, "human_sexuality": 0.4230769230769231, "college_medicine": 0.43023255813953487, "high_school_government_and_politics": 0.5260416666666666, "college_chemistry": 0.3333333333333333, "logical_fallacies": 0.5308641975308642, "high_school_geography": 0.5076142131979695, "elementary_mathematics": 0.2838196286472148, "human_aging": 0.40540540540540543, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.5477941176470589, "formal_logic": 0.304, "high_school_statistics": 0.33488372093023255, "international_law": 0.525, "high_school_mathematics": 0.2899628252788104, "high_school_computer_science": 0.42424242424242425, "conceptual_physics": 0.32905982905982906, "miscellaneous": 0.5025575447570333, "high_school_chemistry": 0.27722772277227725, "marketing": 0.703862660944206, "professional_law": 0.33268101761252444, "management": 0.6078431372549019, "college_physics": 0.33663366336633666, "jurisprudence": 0.4672897196261682, "world_religions": 0.38235294117647056, "sociology": 0.575, "us_foreign_policy": 0.6161616161616161, "high_school_macroeconomics": 0.4241645244215938, "computer_security": 0.5050505050505051, "moral_scenarios": 0.27293064876957496, "moral_disputes": 0.5043478260869565, "electrical_engineering": 0.4027777777777778, "astronomy": 0.40397350993377484, "college_biology": 0.34265734265734266 } }, "prompt_3": { "accuracy": 0.41623167679656775, "category_acc": { "high_school_european_history": 0.5792682926829268, "business_ethics": 0.5757575757575758, "clinical_knowledge": 0.4583333333333333, "medical_genetics": 0.40404040404040403, "high_school_us_history": 0.5467980295566502, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.5932203389830508, "virology": 0.3696969696969697, "high_school_microeconomics": 0.45147679324894513, "econometrics": 0.25663716814159293, "college_computer_science": 0.3333333333333333, "high_school_biology": 0.42394822006472493, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.3736654804270463, "philosophy": 0.432258064516129, "professional_medicine": 0.4022140221402214, "nutrition": 0.46885245901639344, "global_facts": 0.2828282828282828, "machine_learning": 0.32432432432432434, "security_studies": 0.5655737704918032, "public_relations": 0.47706422018348627, "professional_psychology": 0.3698854337152209, "prehistory": 0.3993808049535604, "anatomy": 0.3880597014925373, "human_sexuality": 0.4153846153846154, "college_medicine": 0.42441860465116277, "high_school_government_and_politics": 0.5260416666666666, "college_chemistry": 0.3333333333333333, "logical_fallacies": 0.5246913580246914, "high_school_geography": 0.5126903553299492, "elementary_mathematics": 0.2838196286472148, "human_aging": 0.4189189189189189, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.5514705882352942, "formal_logic": 0.328, "high_school_statistics": 0.32558139534883723, "international_law": 0.5416666666666666, "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.37373737373737376, "conceptual_physics": 0.33760683760683763, "miscellaneous": 0.5, "high_school_chemistry": 0.31683168316831684, "marketing": 0.703862660944206, "professional_law": 0.33920417482061316, "management": 0.6470588235294118, "college_physics": 0.3465346534653465, "jurisprudence": 0.4485981308411215, "world_religions": 0.4, "sociology": 0.585, "us_foreign_policy": 0.6161616161616161, "high_school_macroeconomics": 0.442159383033419, "computer_security": 0.5050505050505051, "moral_scenarios": 0.26174496644295303, "moral_disputes": 0.5072463768115942, "electrical_engineering": 0.4027777777777778, "astronomy": 0.3973509933774834, "college_biology": 0.3146853146853147 } }, "prompt_4": { "accuracy": 0.4138720057204147, "category_acc": { "high_school_european_history": 0.5487804878048781, "business_ethics": 0.5050505050505051, "clinical_knowledge": 0.45454545454545453, "medical_genetics": 0.40404040404040403, "high_school_us_history": 0.5517241379310345, "high_school_physics": 0.3, "high_school_world_history": 0.5889830508474576, "virology": 0.36363636363636365, "high_school_microeconomics": 0.45147679324894513, "econometrics": 0.23893805309734514, "college_computer_science": 0.3333333333333333, "high_school_biology": 0.44660194174757284, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.36298932384341637, "philosophy": 0.41935483870967744, "professional_medicine": 0.4059040590405904, "nutrition": 0.46557377049180326, "global_facts": 0.2828282828282828, "machine_learning": 0.2702702702702703, "security_studies": 0.5614754098360656, "public_relations": 0.44036697247706424, "professional_psychology": 0.3617021276595745, "prehistory": 0.3993808049535604, "anatomy": 0.40298507462686567, "human_sexuality": 0.4230769230769231, "college_medicine": 0.4011627906976744, "high_school_government_and_politics": 0.546875, "college_chemistry": 0.3333333333333333, "logical_fallacies": 0.5308641975308642, "high_school_geography": 0.5279187817258884, "elementary_mathematics": 0.2864721485411141, "human_aging": 0.4144144144144144, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.5551470588235294, "formal_logic": 0.304, "high_school_statistics": 0.34418604651162793, "international_law": 0.5333333333333333, "high_school_mathematics": 0.2899628252788104, "high_school_computer_science": 0.37373737373737376, "conceptual_physics": 0.33760683760683763, "miscellaneous": 0.5127877237851662, "high_school_chemistry": 0.2871287128712871, "marketing": 0.6995708154506438, "professional_law": 0.33463796477495106, "management": 0.6176470588235294, "college_physics": 0.3465346534653465, "jurisprudence": 0.4672897196261682, "world_religions": 0.38235294117647056, "sociology": 0.59, "us_foreign_policy": 0.6262626262626263, "high_school_macroeconomics": 0.42930591259640105, "computer_security": 0.48484848484848486, "moral_scenarios": 0.25727069351230425, "moral_disputes": 0.5101449275362319, "electrical_engineering": 0.4097222222222222, "astronomy": 0.4105960264900662, "college_biology": 0.34265734265734266 } }, "prompt_5": { "accuracy": 0.4199499463711119, "category_acc": { "high_school_european_history": 0.573170731707317, "business_ethics": 0.5252525252525253, "clinical_knowledge": 0.45075757575757575, "medical_genetics": 0.42424242424242425, "high_school_us_history": 0.5467980295566502, "high_school_physics": 0.31333333333333335, "high_school_world_history": 0.597457627118644, "virology": 0.38181818181818183, "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.23893805309734514, "college_computer_science": 0.29292929292929293, "high_school_biology": 0.44660194174757284, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.3736654804270463, "philosophy": 0.43548387096774194, "professional_medicine": 0.3800738007380074, "nutrition": 0.4721311475409836, "global_facts": 0.29292929292929293, "machine_learning": 0.3153153153153153, "security_studies": 0.5614754098360656, "public_relations": 0.46788990825688076, "professional_psychology": 0.3862520458265139, "prehistory": 0.4179566563467492, "anatomy": 0.417910447761194, "human_sexuality": 0.4153846153846154, "college_medicine": 0.436046511627907, "high_school_government_and_politics": 0.53125, "college_chemistry": 0.32323232323232326, "logical_fallacies": 0.5555555555555556, "high_school_geography": 0.5482233502538071, "elementary_mathematics": 0.2864721485411141, "human_aging": 0.42342342342342343, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.5588235294117647, "formal_logic": 0.304, "high_school_statistics": 0.32558139534883723, "international_law": 0.5666666666666667, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.3939393939393939, "conceptual_physics": 0.3333333333333333, "miscellaneous": 0.5089514066496164, "high_school_chemistry": 0.30198019801980197, "marketing": 0.7339055793991416, "professional_law": 0.33659491193737767, "management": 0.6078431372549019, "college_physics": 0.3564356435643564, "jurisprudence": 0.5046728971962616, "world_religions": 0.3764705882352941, "sociology": 0.595, "us_foreign_policy": 0.6161616161616161, "high_school_macroeconomics": 0.42930591259640105, "computer_security": 0.5151515151515151, "moral_scenarios": 0.27628635346756153, "moral_disputes": 0.5014492753623189, "electrical_engineering": 0.4166666666666667, "astronomy": 0.39072847682119205, "college_biology": 0.34965034965034963 } } }, "c_eval": { "prompt_1": { "accuracy": 0.24962852897473997 }, "prompt_2": { "accuracy": 0.25482912332838037 }, "prompt_3": { "accuracy": 0.2473997028231798 }, "prompt_4": { "accuracy": 0.2451708766716196 }, "prompt_5": { "accuracy": 0.2473997028231798 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.24719800747198006, "category_acc": { "computer_network": 0.25, "operating_system": 0.16666666666666666, "computer_architecture": 0.3076923076923077, "college_programming": 0.19047619047619047, "college_physics": 0.041666666666666664, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.375, "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.21428571428571427, "college_economics": 0.31666666666666665, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.23529411764705882, "teacher_qualification": 0.30612244897959184, "high_school_politics": 0.20833333333333334, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.19230769230769232, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.07142857142857142, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.25925925925925924, "law": 0.2413793103448276, "chinese_language_and_literature": 0.25, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.20588235294117646, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.12, "middle_school_history": 0.2962962962962963, "civil_servant": 0.23076923076923078, "sports_science": 0.25, "plant_protection": 0.2222222222222222, "basic_medicine": 0.125, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.18518518518518517, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.2962962962962963, "physician": 0.2777777777777778 } }, "prompt_2": { "accuracy": 0.2559153175591532, "category_acc": { "computer_network": 0.25, "operating_system": 0.125, "computer_architecture": 0.34615384615384615, "college_programming": 0.21428571428571427, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.19047619047619047, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.375, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.25, "college_economics": 0.3333333333333333, "business_administration": 0.18421052631578946, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.3448275862068966, "education_science": 0.2647058823529412, "teacher_qualification": 0.30612244897959184, "high_school_politics": 0.125, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.17647058823529413, "modern_chinese_history": 0.17857142857142858, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.25925925925925924, "law": 0.2413793103448276, "chinese_language_and_literature": 0.25, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.23529411764705882, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.25, "high_school_history": 0.2, "middle_school_history": 0.25925925925925924, "civil_servant": 0.19230769230769232, "sports_science": 0.25, "plant_protection": 0.25925925925925924, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.2037037037037037, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.25925925925925924, "physician": 0.2962962962962963 } }, "prompt_3": { "accuracy": 0.24782067247820672, "category_acc": { "computer_network": 0.25, "operating_system": 0.16666666666666666, "computer_architecture": 0.23076923076923078, "college_programming": 0.2619047619047619, "college_physics": 0.08333333333333333, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.25, "college_economics": 0.2833333333333333, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.3448275862068966, "education_science": 0.2647058823529412, "teacher_qualification": 0.32653061224489793, "high_school_politics": 0.125, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.19230769230769232, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.14285714285714285, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.2962962962962963, "law": 0.2413793103448276, "chinese_language_and_literature": 0.25, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.20588235294117646, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.16, "middle_school_history": 0.25925925925925924, "civil_servant": 0.23076923076923078, "sports_science": 0.16666666666666666, "plant_protection": 0.2222222222222222, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.14814814814814814, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.24074074074074073, "physician": 0.2777777777777778 } }, "prompt_4": { "accuracy": 0.24097135740971357, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.125, "computer_architecture": 0.23076923076923078, "college_programming": 0.2857142857142857, "college_physics": 0.16666666666666666, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.42857142857142855, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.13793103448275862, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.25, "high_school_biology": 0.25, "middle_school_mathematics": 0.125, "middle_school_biology": 0.19230769230769232, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.2, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.31666666666666665, "business_administration": 0.23684210526315788, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.2647058823529412, "teacher_qualification": 0.24489795918367346, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.15384615384615385, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.07142857142857142, "ideological_and_moral_cultivation": 0.25, "logic": 0.25925925925925924, "law": 0.27586206896551724, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.47368421052631576, "professional_tour_guide": 0.23529411764705882, "legal_professional": 0.25, "high_school_chinese": 0.25, "high_school_history": 0.12, "middle_school_history": 0.3333333333333333, "civil_servant": 0.19230769230769232, "sports_science": 0.08333333333333333, "plant_protection": 0.1111111111111111, "basic_medicine": 0.16666666666666666, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.1111111111111111, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.25925925925925924, "physician": 0.2962962962962963 } }, "prompt_5": { "accuracy": 0.2521793275217933, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.16666666666666666, "computer_architecture": 0.2692307692307692, "college_programming": 0.23809523809523808, "college_physics": 0.16666666666666666, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.4583333333333333, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.16666666666666666, "metrology_engineer": 0.13793103448275862, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.375, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.21428571428571427, "college_economics": 0.31666666666666665, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.3448275862068966, "education_science": 0.23529411764705882, "teacher_qualification": 0.3673469387755102, "high_school_politics": 0.20833333333333334, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.2857142857142857, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.25925925925925924, "law": 0.2413793103448276, "chinese_language_and_literature": 0.25, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.23529411764705882, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.16, "middle_school_history": 0.25925925925925924, "civil_servant": 0.2692307692307692, "sports_science": 0.16666666666666666, "plant_protection": 0.18518518518518517, "basic_medicine": 0.125, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.16666666666666666, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.25925925925925924, "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.2831541218637993 }, "prompt_2": { "accuracy": 0.2867383512544803 }, "prompt_3": { "accuracy": 0.2724014336917563 }, "prompt_4": { "accuracy": 0.26523297491039427 }, "prompt_5": { "accuracy": 0.2724014336917563 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2515109652909687, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.25675675675675674, "ancient_chinese": 0.23780487804878048, "arts": 0.25625, "astronomy": 0.296969696969697, "business_ethics": 0.2583732057416268, "chinese_civil_service_exam": 0.225, "chinese_driving_rule": 0.29770992366412213, "chinese_food_culture": 0.20588235294117646, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.25696594427244585, "chinese_literature": 0.2549019607843137, "chinese_teacher_qualification": 0.22346368715083798, "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.19811320754716982, "college_education": 0.2897196261682243, "college_engineering_hydrology": 0.29245283018867924, "college_law": 0.24074074074074073, "college_mathematics": 0.24761904761904763, "college_medical_statistics": 0.20754716981132076, "college_medicine": 0.2454212454212454, "computer_science": 0.27941176470588236, "computer_security": 0.21637426900584794, "conceptual_physics": 0.22448979591836735, "construction_project_management": 0.23741007194244604, "economics": 0.23270440251572327, "education": 0.26380368098159507, "electrical_engineering": 0.29069767441860467, "elementary_chinese": 0.2857142857142857, "elementary_commonsense": 0.25757575757575757, "elementary_information_and_technology": 0.2647058823529412, "elementary_mathematics": 0.25217391304347825, "ethnology": 0.26666666666666666, "food_science": 0.27972027972027974, "genetics": 0.25, "global_facts": 0.2483221476510067, "high_school_biology": 0.2603550295857988, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.2457627118644068, "high_school_mathematics": 0.27439024390243905, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.25874125874125875, "human_sexuality": 0.24603174603174602, "international_law": 0.2594594594594595, "journalism": 0.2616279069767442, "jurisprudence": 0.25790754257907544, "legal_and_moral_basis": 0.2616822429906542, "logical": 0.2764227642276423, "machine_learning": 0.2459016393442623, "management": 0.22380952380952382, "marketing": 0.2, "marxist_theory": 0.2222222222222222, "modern_chinese": 0.2413793103448276, "nutrition": 0.2689655172413793, "philosophy": 0.3238095238095238, "professional_accounting": 0.24, "professional_law": 0.25118483412322273, "professional_medicine": 0.24202127659574468, "professional_psychology": 0.24568965517241378, "public_relations": 0.2471264367816092, "security_study": 0.23703703703703705, "sociology": 0.22566371681415928, "sports_science": 0.24848484848484848, "traditional_chinese_medicine": 0.2810810810810811, "virology": 0.24260355029585798, "world_history": 0.21739130434782608, "world_religions": 0.2625 } }, "prompt_2": { "accuracy": 0.25090657917458126, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.27702702702702703, "ancient_chinese": 0.24390243902439024, "arts": 0.25, "astronomy": 0.23030303030303031, "business_ethics": 0.2631578947368421, "chinese_civil_service_exam": 0.25625, "chinese_driving_rule": 0.26717557251908397, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.29906542056074764, "chinese_history": 0.24148606811145512, "chinese_literature": 0.2647058823529412, "chinese_teacher_qualification": 0.2346368715083799, "clinical_knowledge": 0.25316455696202533, "college_actuarial_science": 0.20754716981132076, "college_education": 0.29906542056074764, "college_engineering_hydrology": 0.24528301886792453, "college_law": 0.19444444444444445, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.18867924528301888, "college_medicine": 0.2454212454212454, "computer_science": 0.27941176470588236, "computer_security": 0.21637426900584794, "conceptual_physics": 0.23809523809523808, "construction_project_management": 0.2517985611510791, "economics": 0.2389937106918239, "education": 0.26380368098159507, "electrical_engineering": 0.2616279069767442, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.25757575757575757, "elementary_information_and_technology": 0.2647058823529412, "elementary_mathematics": 0.2826086956521739, "ethnology": 0.2740740740740741, "food_science": 0.23776223776223776, "genetics": 0.23863636363636365, "global_facts": 0.24161073825503357, "high_school_biology": 0.2485207100591716, "high_school_chemistry": 0.3181818181818182, "high_school_geography": 0.2796610169491525, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.2545454545454545, "high_school_politics": 0.24475524475524477, "human_sexuality": 0.25396825396825395, "international_law": 0.2648648648648649, "journalism": 0.26744186046511625, "jurisprudence": 0.25304136253041365, "legal_and_moral_basis": 0.24299065420560748, "logical": 0.2682926829268293, "machine_learning": 0.2459016393442623, "management": 0.21428571428571427, "marketing": 0.22777777777777777, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25862068965517243, "nutrition": 0.2413793103448276, "philosophy": 0.26666666666666666, "professional_accounting": 0.26285714285714284, "professional_law": 0.26540284360189575, "professional_medicine": 0.2393617021276596, "professional_psychology": 0.25, "public_relations": 0.23563218390804597, "security_study": 0.21481481481481482, "sociology": 0.25663716814159293, "sports_science": 0.22424242424242424, "traditional_chinese_medicine": 0.2756756756756757, "virology": 0.23668639053254437, "world_history": 0.2484472049689441, "world_religions": 0.225 } }, "prompt_3": { "accuracy": 0.25125194266965983, "category_acc": { "agronomy": 0.21893491124260356, "anatomy": 0.2702702702702703, "ancient_chinese": 0.25, "arts": 0.25625, "astronomy": 0.24848484848484848, "business_ethics": 0.2679425837320574, "chinese_civil_service_exam": 0.2125, "chinese_driving_rule": 0.2748091603053435, "chinese_food_culture": 0.25735294117647056, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.23839009287925697, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.2358490566037736, "college_education": 0.2897196261682243, "college_engineering_hydrology": 0.3018867924528302, "college_law": 0.23148148148148148, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.22641509433962265, "college_medicine": 0.2490842490842491, "computer_science": 0.22058823529411764, "computer_security": 0.21637426900584794, "conceptual_physics": 0.24489795918367346, "construction_project_management": 0.28776978417266186, "economics": 0.2389937106918239, "education": 0.26380368098159507, "electrical_engineering": 0.27325581395348836, "elementary_chinese": 0.2896825396825397, "elementary_commonsense": 0.24242424242424243, "elementary_information_and_technology": 0.2689075630252101, "elementary_mathematics": 0.22608695652173913, "ethnology": 0.25925925925925924, "food_science": 0.2937062937062937, "genetics": 0.23863636363636365, "global_facts": 0.2550335570469799, "high_school_biology": 0.2603550295857988, "high_school_chemistry": 0.2878787878787879, "high_school_geography": 0.2627118644067797, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.2727272727272727, "human_sexuality": 0.2698412698412698, "international_law": 0.2594594594594595, "journalism": 0.28488372093023256, "jurisprudence": 0.24330900243309003, "legal_and_moral_basis": 0.24299065420560748, "logical": 0.22764227642276422, "machine_learning": 0.2459016393442623, "management": 0.24285714285714285, "marketing": 0.2222222222222222, "marxist_theory": 0.2222222222222222, "modern_chinese": 0.27586206896551724, "nutrition": 0.2689655172413793, "philosophy": 0.29523809523809524, "professional_accounting": 0.24, "professional_law": 0.27014218009478674, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.2413793103448276, "public_relations": 0.22988505747126436, "security_study": 0.21481481481481482, "sociology": 0.22566371681415928, "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.2918918918918919, "virology": 0.23076923076923078, "world_history": 0.18012422360248448, "world_religions": 0.2625 } }, "prompt_4": { "accuracy": 0.2493524434467277, "category_acc": { "agronomy": 0.23076923076923078, "anatomy": 0.2635135135135135, "ancient_chinese": 0.23780487804878048, "arts": 0.26875, "astronomy": 0.2727272727272727, "business_ethics": 0.2583732057416268, "chinese_civil_service_exam": 0.21875, "chinese_driving_rule": 0.2595419847328244, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.205607476635514, "chinese_history": 0.21981424148606812, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.24581005586592178, "clinical_knowledge": 0.2742616033755274, "college_actuarial_science": 0.2358490566037736, "college_education": 0.308411214953271, "college_engineering_hydrology": 0.3113207547169811, "college_law": 0.21296296296296297, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.2169811320754717, "college_medicine": 0.25274725274725274, "computer_science": 0.23039215686274508, "computer_security": 0.24561403508771928, "conceptual_physics": 0.2585034013605442, "construction_project_management": 0.2589928057553957, "economics": 0.2389937106918239, "education": 0.2392638036809816, "electrical_engineering": 0.2441860465116279, "elementary_chinese": 0.24603174603174602, "elementary_commonsense": 0.25757575757575757, "elementary_information_and_technology": 0.25630252100840334, "elementary_mathematics": 0.29130434782608694, "ethnology": 0.2814814814814815, "food_science": 0.25874125874125875, "genetics": 0.23295454545454544, "global_facts": 0.24161073825503357, "high_school_biology": 0.25443786982248523, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.22727272727272727, "high_school_politics": 0.23076923076923078, "human_sexuality": 0.25396825396825395, "international_law": 0.24864864864864866, "journalism": 0.2558139534883721, "jurisprudence": 0.2384428223844282, "legal_and_moral_basis": 0.2336448598130841, "logical": 0.2601626016260163, "machine_learning": 0.20491803278688525, "management": 0.24285714285714285, "marketing": 0.25, "marxist_theory": 0.24338624338624337, "modern_chinese": 0.22413793103448276, "nutrition": 0.2689655172413793, "philosophy": 0.2857142857142857, "professional_accounting": 0.24571428571428572, "professional_law": 0.25118483412322273, "professional_medicine": 0.2393617021276596, "professional_psychology": 0.2413793103448276, "public_relations": 0.2413793103448276, "security_study": 0.23703703703703705, "sociology": 0.26548672566371684, "sports_science": 0.2545454545454545, "traditional_chinese_medicine": 0.2918918918918919, "virology": 0.24260355029585798, "world_history": 0.2546583850931677, "world_religions": 0.2625 } }, "prompt_5": { "accuracy": 0.25116560179589015, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.2635135135135135, "ancient_chinese": 0.2682926829268293, "arts": 0.275, "astronomy": 0.23030303030303031, "business_ethics": 0.2631578947368421, "chinese_civil_service_exam": 0.25, "chinese_driving_rule": 0.26717557251908397, "chinese_food_culture": 0.20588235294117646, "chinese_foreign_policy": 0.21495327102803738, "chinese_history": 0.26006191950464397, "chinese_literature": 0.24509803921568626, "chinese_teacher_qualification": 0.21787709497206703, "clinical_knowledge": 0.24050632911392406, "college_actuarial_science": 0.19811320754716982, "college_education": 0.2803738317757009, "college_engineering_hydrology": 0.25471698113207547, "college_law": 0.17592592592592593, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.1792452830188679, "college_medicine": 0.23076923076923078, "computer_science": 0.2696078431372549, "computer_security": 0.2046783625730994, "conceptual_physics": 0.2585034013605442, "construction_project_management": 0.2589928057553957, "economics": 0.2578616352201258, "education": 0.26380368098159507, "electrical_engineering": 0.28488372093023256, "elementary_chinese": 0.30158730158730157, "elementary_commonsense": 0.26262626262626265, "elementary_information_and_technology": 0.24789915966386555, "elementary_mathematics": 0.2782608695652174, "ethnology": 0.3111111111111111, "food_science": 0.25874125874125875, "genetics": 0.2727272727272727, "global_facts": 0.2348993288590604, "high_school_biology": 0.24260355029585798, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.2457627118644068, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.2727272727272727, "high_school_politics": 0.23776223776223776, "human_sexuality": 0.23809523809523808, "international_law": 0.25405405405405407, "journalism": 0.27325581395348836, "jurisprudence": 0.25304136253041365, "legal_and_moral_basis": 0.27102803738317754, "logical": 0.25203252032520324, "machine_learning": 0.27049180327868855, "management": 0.22857142857142856, "marketing": 0.2611111111111111, "marxist_theory": 0.2275132275132275, "modern_chinese": 0.25862068965517243, "nutrition": 0.2620689655172414, "philosophy": 0.29523809523809524, "professional_accounting": 0.24, "professional_law": 0.27014218009478674, "professional_medicine": 0.23404255319148937, "professional_psychology": 0.22844827586206898, "public_relations": 0.22413793103448276, "security_study": 0.22962962962962963, "sociology": 0.2345132743362832, "sports_science": 0.23030303030303031, "traditional_chinese_medicine": 0.3027027027027027, "virology": 0.24260355029585798, "world_history": 0.2484472049689441, "world_religions": 0.25 } } }, "zbench": { "prompt_1": { "accuracy": 0.36363636363636365 }, "prompt_2": { "accuracy": 0.2727272727272727 }, "prompt_3": { "accuracy": 0.30303030303030304 }, "prompt_4": { "accuracy": 0.2727272727272727 }, "prompt_5": { "accuracy": 0.30303030303030304 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.2727272727272727 }, "prompt_2": { "accuracy": 0.30227272727272725 }, "prompt_3": { "accuracy": 0.23636363636363636 }, "prompt_4": { "accuracy": 0.325 }, "prompt_5": { "accuracy": 0.2840909090909091 } }, "ocnli": { "prompt_1": { "accuracy": 0.3305084745762712 }, "prompt_2": { "accuracy": 0.32 }, "prompt_3": { "accuracy": 0.3342372881355932 }, "prompt_4": { "accuracy": 0.3176271186440678 }, "prompt_5": { "accuracy": 0.3342372881355932 } }, "c3": { "prompt_1": { "accuracy": 0.26028421839940163 }, "prompt_2": { "accuracy": 0.2655198204936425 }, "prompt_3": { "accuracy": 0.27786088257292446 }, "prompt_4": { "accuracy": 0.2801047120418848 }, "prompt_5": { "accuracy": 0.27599102468212416 } }, "dream": { "prompt_1": { "accuracy": 0.875061244487996 }, "prompt_2": { "accuracy": 0.8682018618324351 }, "prompt_3": { "accuracy": 0.8740813326800588 }, "prompt_4": { "accuracy": 0.8726114649681529 }, "prompt_5": { "accuracy": 0.8716315531602156 } }, "samsum": { "prompt_1": { "rouge1": 0.5185659226290711, "rouge2": 0.26879953180572064, "rougeL": 0.43275638022449625, "avg_rouge": 0.40670727821976266 }, "prompt_2": { "rouge1": 0.5190628568296515, "rouge2": 0.26969716664146526, "rougeL": 0.4317910039013235, "avg_rouge": 0.4068503424574801 }, "prompt_3": { "rouge1": 0.5164177682133352, "rouge2": 0.2634396180719505, "rougeL": 0.4281279644017568, "avg_rouge": 0.40266178356234744 }, "prompt_4": { "rouge1": 0.518744098149516, "rouge2": 0.2687583498609372, "rougeL": 0.4321035265896283, "avg_rouge": 0.40653532486669386 }, "prompt_5": { "rouge1": 0.5175029512092139, "rouge2": 0.26827791419756786, "rougeL": 0.4309740774155282, "avg_rouge": 0.40558498094076995 } }, "dialogsum": { "prompt_1": { "rouge1": 0.24423627199761502, "rouge2": 0.07472364536524811, "rougeL": 0.18416102098715098, "avg_rouge": 0.1677069794500047 }, "prompt_2": { "rouge1": 0.2438277162660386, "rouge2": 0.07408111256345526, "rougeL": 0.18586363370004302, "avg_rouge": 0.1679241541765123 }, "prompt_3": { "rouge1": 0.23272168097538612, "rouge2": 0.06643126452079036, "rougeL": 0.17368331730278494, "avg_rouge": 0.1576120875996538 }, "prompt_4": { "rouge1": 0.2331686673676627, "rouge2": 0.06578694871923757, "rougeL": 0.1738804407479083, "avg_rouge": 0.15761201894493618 }, "prompt_5": { "rouge1": 0.24261531328574346, "rouge2": 0.07032891941011479, "rougeL": 0.18516565082477832, "avg_rouge": 0.1660366278402122 } }, "sst2": { "prompt_1": { "accuracy": 0.9529816513761468 }, "prompt_2": { "accuracy": 0.9506880733944955 }, "prompt_3": { "accuracy": 0.9495412844036697 }, "prompt_4": { "accuracy": 0.9529816513761468 }, "prompt_5": { "accuracy": 0.948394495412844 } }, "cola": { "prompt_1": { "accuracy": 0.7171620325982742 }, "prompt_2": { "accuracy": 0.7277085330776606 }, "prompt_3": { "accuracy": 0.7229146692233941 }, "prompt_4": { "accuracy": 0.713326941514861 }, "prompt_5": { "accuracy": 0.7248322147651006 } }, "qqp": { "prompt_1": { "accuracy": 0.824 }, "prompt_2": { "accuracy": 0.853 }, "prompt_3": { "accuracy": 0.821 }, "prompt_4": { "accuracy": 0.8275 }, "prompt_5": { "accuracy": 0.8465 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.7 } }, "qnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.5 } }, "rte": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 1.0 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "flan_t5_xl": { "model_size": "3B", "model_link": "https://huggingface.co/google/flan-t5-xl", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.7733193277310924, "language_acc": { "Spanish": 0.9563025210084034, "English": 0.9697478991596639, "Chinese": 0.3563025210084034, "Vietnamese": 0.8109243697478992 }, "consistency_score_2": 0.6148459383753501, "consistency_score_3": 0.4455882352941176, "consistency_score_4": 0.3142857142857143, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9537815126050421, "Spanish,Chinese": 0.3621848739495798, "Spanish,Vietnamese": 0.807563025210084, "English,Chinese": 0.3621848739495798, "English,Vietnamese": 0.8142857142857143, "Chinese,Vietnamese": 0.38907563025210085 }, "3_combine": { "Spanish,English,Chinese": 0.3495798319327731, "Spanish,English,Vietnamese": 0.7915966386554621, "Spanish,Chinese,Vietnamese": 0.32016806722689073, "English,Chinese,Vietnamese": 0.32100840336134456 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3142857142857143 } }, "AC3_2": 0.6850369466770796, "AC3_3": 0.5653947928242272, "AC3_4": 0.4469328623447341 }, "prompt_2": { "overall_acc": 0.7762605042016806, "language_acc": { "Spanish": 0.9588235294117647, "English": 0.9697478991596639, "Chinese": 0.36470588235294116, "Vietnamese": 0.8117647058823529 }, "consistency_score_2": 0.6179271708683473, "consistency_score_3": 0.4521008403361344, "consistency_score_4": 0.3226890756302521, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9529411764705882, "Spanish,Chinese": 0.37058823529411766, "Spanish,Vietnamese": 0.807563025210084, "English,Chinese": 0.37142857142857144, "English,Vietnamese": 0.8117647058823529, "Chinese,Vietnamese": 0.39327731092436974 }, "3_combine": { "Spanish,English,Chinese": 0.35798319327731093, "Spanish,English,Vietnamese": 0.7907563025210084, "Spanish,Chinese,Vietnamese": 0.32941176470588235, "English,Chinese,Vietnamese": 0.3302521008403361 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3226890756302521 } }, "AC3_2": 0.6881031381369555, "AC3_3": 0.5714084504552835, "AC3_4": 0.4558731157886766 }, "prompt_3": { "overall_acc": 0.774579831932773, "language_acc": { "Spanish": 0.9554621848739496, "English": 0.9697478991596639, "Chinese": 0.3621848739495798, "Vietnamese": 0.8109243697478992 }, "consistency_score_2": 0.6170868347338935, "consistency_score_3": 0.4491596638655462, "consistency_score_4": 0.31848739495798317, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9529411764705882, "Spanish,Chinese": 0.36722689075630255, "Spanish,Vietnamese": 0.8084033613445378, "English,Chinese": 0.3689075630252101, "English,Vietnamese": 0.8117647058823529, "Chinese,Vietnamese": 0.39327731092436974 }, "3_combine": { "Spanish,English,Chinese": 0.3546218487394958, "Spanish,English,Vietnamese": 0.7907563025210084, "Spanish,Chinese,Vietnamese": 0.32521008403361346, "English,Chinese,Vietnamese": 0.32605042016806723 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.31848739495798317 } }, "AC3_2": 0.686921700649236, "AC3_3": 0.5686014353774906, "AC3_4": 0.4513792139545092 }, "prompt_4": { "overall_acc": 0.7731092436974789, "language_acc": { "Spanish": 0.9571428571428572, "English": 0.9680672268907563, "Chinese": 0.36554621848739494, "Vietnamese": 0.8016806722689076 }, "consistency_score_2": 0.6168067226890755, "consistency_score_3": 0.45042016806722684, "consistency_score_4": 0.32184873949579834, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9529411764705882, "Spanish,Chinese": 0.37058823529411766, "Spanish,Vietnamese": 0.8016806722689076, "English,Chinese": 0.3722689075630252, "English,Vietnamese": 0.8033613445378152, "Chinese,Vietnamese": 0.4 }, "3_combine": { "Spanish,English,Chinese": 0.3588235294117647, "Spanish,English,Vietnamese": 0.7840336134453781, "Spanish,Chinese,Vietnamese": 0.32941176470588235, "English,Chinese,Vietnamese": 0.32941176470588235 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.32184873949579834 } }, "AC3_2": 0.686169510076327, "AC3_3": 0.5692123002582151, "AC3_4": 0.45449092910067884 }, "prompt_5": { "overall_acc": 0.7752100840336134, "language_acc": { "Spanish": 0.957983193277311, "English": 0.9722689075630252, "Chinese": 0.3563025210084034, "Vietnamese": 0.8142857142857143 }, "consistency_score_2": 0.6134453781512604, "consistency_score_3": 0.4426470588235294, "consistency_score_4": 0.30756302521008405, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9554621848739496, "Spanish,Chinese": 0.35798319327731093, "Spanish,Vietnamese": 0.8142857142857143, "English,Chinese": 0.3596638655462185, "English,Vietnamese": 0.8168067226890756, "Chinese,Vietnamese": 0.3764705882352941 }, "3_combine": { "Spanish,English,Chinese": 0.3445378151260504, "Spanish,English,Vietnamese": 0.7983193277310925, "Spanish,Chinese,Vietnamese": 0.3142857142857143, "English,Chinese,Vietnamese": 0.3134453781512605 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.30756302521008405 } }, "AC3_2": 0.6849057322889995, "AC3_3": 0.5635216998187702, "AC3_4": 0.44039874385735167 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.36, "language_acc": { "Filipino": 0.2866666666666667, "Vietnamese": 0.24666666666666667, "Chinese": 0.26666666666666666, "Spanish": 0.44666666666666666, "Malay": 0.34, "Indonesian": 0.37333333333333335, "English": 0.56 }, "consistency_score_2": 0.5149206349206349, "consistency_score_3": 0.3361904761904762, "consistency_score_4": 0.2432380952380952, "consistency_score_5": 0.1853968253968254, "consistency_score_6": 0.1466666666666667, "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5066666666666667, "Filipino,Chinese": 0.5066666666666667, "Filipino,Spanish": 0.49333333333333335, "Filipino,Malay": 0.5866666666666667, "Filipino,Indonesian": 0.5266666666666666, "Filipino,English": 0.5133333333333333, "Vietnamese,Chinese": 0.66, "Vietnamese,Spanish": 0.44, "Vietnamese,Malay": 0.5266666666666666, "Vietnamese,Indonesian": 0.5266666666666666, "Vietnamese,English": 0.36666666666666664, "Chinese,Spanish": 0.4266666666666667, "Chinese,Malay": 0.5533333333333333, "Chinese,Indonesian": 0.5666666666666667, "Chinese,English": 0.3466666666666667, "Spanish,Malay": 0.5, "Spanish,Indonesian": 0.5466666666666666, "Spanish,English": 0.5866666666666667, "Malay,Indonesian": 0.6666666666666666, "Malay,English": 0.48, "Indonesian,English": 0.4866666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.38, "Filipino,Vietnamese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Malay": 0.37333333333333335, "Filipino,Vietnamese,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,English": 0.26, "Filipino,Chinese,Spanish": 0.26666666666666666, "Filipino,Chinese,Malay": 0.38666666666666666, "Filipino,Chinese,Indonesian": 0.35333333333333333, "Filipino,Chinese,English": 0.26666666666666666, "Filipino,Spanish,Malay": 0.36, "Filipino,Spanish,Indonesian": 0.34, "Filipino,Spanish,English": 0.36666666666666664, "Filipino,Malay,Indonesian": 0.4533333333333333, "Filipino,Malay,English": 0.35333333333333333, "Filipino,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish": 0.32, "Vietnamese,Chinese,Malay": 0.42, "Vietnamese,Chinese,Indonesian": 0.4266666666666667, "Vietnamese,Chinese,English": 0.26, "Vietnamese,Spanish,Malay": 0.31333333333333335, "Vietnamese,Spanish,Indonesian": 0.3333333333333333, "Vietnamese,Spanish,English": 0.2733333333333333, "Vietnamese,Malay,Indonesian": 0.4, "Vietnamese,Malay,English": 0.26666666666666666, "Vietnamese,Indonesian,English": 0.2733333333333333, "Chinese,Spanish,Malay": 0.29333333333333333, "Chinese,Spanish,Indonesian": 0.32666666666666666, "Chinese,Spanish,English": 0.26, "Chinese,Malay,Indonesian": 0.44, "Chinese,Malay,English": 0.26666666666666666, "Chinese,Indonesian,English": 0.29333333333333333, "Spanish,Malay,Indonesian": 0.41333333333333333, "Spanish,Malay,English": 0.35333333333333333, "Spanish,Indonesian,English": 0.36, "Malay,Indonesian,English": 0.36666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.2, "Filipino,Vietnamese,Chinese,Malay": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Chinese,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, "Filipino,Chinese,Malay,English": 0.22666666666666666, "Filipino,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.3, "Filipino,Spanish,Malay,English": 0.2733333333333333, "Filipino,Spanish,Indonesian,English": 0.26, "Filipino,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, "Vietnamese,Chinese,Malay,English": 0.2, "Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, "Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.22, "Vietnamese,Malay,Indonesian,English": 0.22, "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, "Chinese,Spanish,Malay,English": 0.20666666666666667, "Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Chinese,Malay,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Malay,English": 0.16, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.18, "Filipino,Chinese,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, "Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, "AC3_2": 0.4237445572810306, "AC3_3": 0.3476880984452705, "AC3_4": 0.2903189137504208, "AC3_5": 0.24474970891903602, "AC3_6": 0.20842105259044322, "AC3_7": 0.1799999999625 }, "prompt_2": { "overall_acc": 0.35809523809523813, "language_acc": { "Filipino": 0.2733333333333333, "Vietnamese": 0.24666666666666667, "Chinese": 0.2866666666666667, "Spanish": 0.4533333333333333, "Malay": 0.36, "Indonesian": 0.38, "English": 0.5066666666666667 }, "consistency_score_2": 0.5292063492063492, "consistency_score_3": 0.3466666666666667, "consistency_score_4": 0.24723809523809526, "consistency_score_5": 0.1841269841269841, "consistency_score_6": 0.13999999999999999, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.52, "Filipino,Chinese": 0.5533333333333333, "Filipino,Spanish": 0.5066666666666667, "Filipino,Malay": 0.62, "Filipino,Indonesian": 0.5666666666666667, "Filipino,English": 0.4866666666666667, "Vietnamese,Chinese": 0.64, "Vietnamese,Spanish": 0.44666666666666666, "Vietnamese,Malay": 0.52, "Vietnamese,Indonesian": 0.54, "Vietnamese,English": 0.36, "Chinese,Spanish": 0.43333333333333335, "Chinese,Malay": 0.5866666666666667, "Chinese,Indonesian": 0.58, "Chinese,English": 0.37333333333333335, "Spanish,Malay": 0.52, "Spanish,Indonesian": 0.5466666666666666, "Spanish,English": 0.6266666666666667, "Malay,Indonesian": 0.6666666666666666, "Malay,English": 0.5, "Indonesian,English": 0.52 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.4, "Filipino,Vietnamese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Malay": 0.37333333333333335, "Filipino,Vietnamese,Indonesian": 0.36, "Filipino,Vietnamese,English": 0.24666666666666667, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.4266666666666667, "Filipino,Chinese,Indonesian": 0.3933333333333333, "Filipino,Chinese,English": 0.2733333333333333, "Filipino,Spanish,Malay": 0.38, "Filipino,Spanish,Indonesian": 0.3466666666666667, "Filipino,Spanish,English": 0.36666666666666664, "Filipino,Malay,Indonesian": 0.47333333333333333, "Filipino,Malay,English": 0.35333333333333333, "Filipino,Indonesian,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish": 0.31333333333333335, "Vietnamese,Chinese,Malay": 0.41333333333333333, "Vietnamese,Chinese,Indonesian": 0.4266666666666667, "Vietnamese,Chinese,English": 0.25333333333333335, "Vietnamese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Spanish,Indonesian": 0.32666666666666666, "Vietnamese,Spanish,English": 0.28, "Vietnamese,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Malay,English": 0.26, "Vietnamese,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay": 0.31333333333333335, "Chinese,Spanish,Indonesian": 0.31333333333333335, "Chinese,Spanish,English": 0.2733333333333333, "Chinese,Malay,Indonesian": 0.46, "Chinese,Malay,English": 0.29333333333333333, "Chinese,Indonesian,English": 0.31333333333333335, "Spanish,Malay,Indonesian": 0.41333333333333333, "Spanish,Malay,English": 0.38666666666666666, "Spanish,Indonesian,English": 0.3933333333333333, "Malay,Indonesian,English": 0.38666666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Chinese,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.22, "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Malay,English": 0.2, "Filipino,Vietnamese,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.35333333333333333, "Filipino,Chinese,Malay,English": 0.22666666666666666, "Filipino,Chinese,Indonesian,English": 0.24, "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, "Filipino,Spanish,Malay,English": 0.28, "Filipino,Spanish,Indonesian,English": 0.26666666666666666, "Filipino,Malay,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,Spanish,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, "Vietnamese,Chinese,Malay,English": 0.2, "Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Chinese,Spanish,Malay,English": 0.22666666666666666, "Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Chinese,Malay,Indonesian,English": 0.25333333333333335, "Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.4271518868246766, "AC3_3": 0.3522882882383015, "AC3_4": 0.2925158080370256, "AC3_5": 0.24320285486760376, "AC3_6": 0.20130019116417489, "AC3_7": 0.16437158466408672 }, "prompt_3": { "overall_acc": 0.35809523809523813, "language_acc": { "Filipino": 0.3, "Vietnamese": 0.25333333333333335, "Chinese": 0.26666666666666666, "Spanish": 0.44, "Malay": 0.32666666666666666, "Indonesian": 0.3933333333333333, "English": 0.5266666666666666 }, "consistency_score_2": 0.540952380952381, "consistency_score_3": 0.36266666666666664, "consistency_score_4": 0.26514285714285707, "consistency_score_5": 0.20412698412698416, "consistency_score_6": 0.16285714285714287, "consistency_score_7": 0.13333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5333333333333333, "Filipino,Chinese": 0.54, "Filipino,Spanish": 0.52, "Filipino,Malay": 0.6666666666666666, "Filipino,Indonesian": 0.6, "Filipino,English": 0.5066666666666667, "Vietnamese,Chinese": 0.66, "Vietnamese,Spanish": 0.44666666666666666, "Vietnamese,Malay": 0.56, "Vietnamese,Indonesian": 0.5666666666666667, "Vietnamese,English": 0.38666666666666666, "Chinese,Spanish": 0.43333333333333335, "Chinese,Malay": 0.5866666666666667, "Chinese,Indonesian": 0.6, "Chinese,English": 0.38, "Spanish,Malay": 0.5133333333333333, "Spanish,Indonesian": 0.5466666666666666, "Spanish,English": 0.5933333333333334, "Malay,Indonesian": 0.7, "Malay,English": 0.4866666666666667, "Indonesian,English": 0.5333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.4, "Filipino,Vietnamese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Malay": 0.42, "Filipino,Vietnamese,Indonesian": 0.38666666666666666, "Filipino,Vietnamese,English": 0.28, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.44, "Filipino,Chinese,Indonesian": 0.41333333333333333, "Filipino,Chinese,English": 0.2866666666666667, "Filipino,Spanish,Malay": 0.3933333333333333, "Filipino,Spanish,Indonesian": 0.37333333333333335, "Filipino,Spanish,English": 0.37333333333333335, "Filipino,Malay,Indonesian": 0.5266666666666666, "Filipino,Malay,English": 0.38, "Filipino,Indonesian,English": 0.37333333333333335, "Vietnamese,Chinese,Spanish": 0.32666666666666666, "Vietnamese,Chinese,Malay": 0.44666666666666666, "Vietnamese,Chinese,Indonesian": 0.46, "Vietnamese,Chinese,English": 0.2866666666666667, "Vietnamese,Spanish,Malay": 0.32, "Vietnamese,Spanish,Indonesian": 0.3333333333333333, "Vietnamese,Spanish,English": 0.2733333333333333, "Vietnamese,Malay,Indonesian": 0.4533333333333333, "Vietnamese,Malay,English": 0.28, "Vietnamese,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay": 0.30666666666666664, "Chinese,Spanish,Indonesian": 0.32666666666666666, "Chinese,Spanish,English": 0.2733333333333333, "Chinese,Malay,Indonesian": 0.48, "Chinese,Malay,English": 0.2866666666666667, "Chinese,Indonesian,English": 0.3333333333333333, "Spanish,Malay,Indonesian": 0.42, "Spanish,Malay,English": 0.36, "Spanish,Indonesian,English": 0.38666666666666666, "Malay,Indonesian,English": 0.4066666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Malay": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Chinese,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.24, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, "Filipino,Chinese,Spanish,Malay": 0.24, "Filipino,Chinese,Spanish,Indonesian": 0.25333333333333335, "Filipino,Chinese,Spanish,English": 0.20666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, "Filipino,Chinese,Malay,English": 0.24, "Filipino,Chinese,Indonesian,English": 0.26666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.3333333333333333, "Filipino,Spanish,Malay,English": 0.2866666666666667, "Filipino,Spanish,Indonesian,English": 0.2866666666666667, "Filipino,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.24, "Vietnamese,Chinese,Spanish,Indonesian": 0.26, "Vietnamese,Chinese,Spanish,English": 0.21333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Chinese,Malay,English": 0.22, "Vietnamese,Chinese,Indonesian,English": 0.26, "Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Malay,Indonesian,English": 0.26, "Chinese,Spanish,Malay,Indonesian": 0.28, "Chinese,Spanish,Malay,English": 0.21333333333333335, "Chinese,Spanish,Indonesian,English": 0.24666666666666667, "Chinese,Malay,Indonesian,English": 0.26666666666666666, "Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.2, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.18, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.2, "Filipino,Chinese,Malay,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 } }, "AC3_2": 0.4309281678293888, "AC3_3": 0.3603664552001782, "AC3_4": 0.3046873907998722, "AC3_5": 0.26002850147293466, "AC3_6": 0.22389135540230864, "AC3_7": 0.1943152454384953 }, "prompt_4": { "overall_acc": 0.3590476190476191, "language_acc": { "Filipino": 0.3, "Vietnamese": 0.24, "Chinese": 0.26, "Spanish": 0.4533333333333333, "Malay": 0.3466666666666667, "Indonesian": 0.38666666666666666, "English": 0.5266666666666666 }, "consistency_score_2": 0.5333333333333334, "consistency_score_3": 0.3523809523809524, "consistency_score_4": 0.25580952380952376, "consistency_score_5": 0.1955555555555556, "consistency_score_6": 0.15333333333333332, "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5333333333333333, "Filipino,Chinese": 0.5466666666666666, "Filipino,Spanish": 0.49333333333333335, "Filipino,Malay": 0.66, "Filipino,Indonesian": 0.5866666666666667, "Filipino,English": 0.4866666666666667, "Vietnamese,Chinese": 0.68, "Vietnamese,Spanish": 0.42, "Vietnamese,Malay": 0.5533333333333333, "Vietnamese,Indonesian": 0.5733333333333334, "Vietnamese,English": 0.36, "Chinese,Spanish": 0.44, "Chinese,Malay": 0.58, "Chinese,Indonesian": 0.5866666666666667, "Chinese,English": 0.37333333333333335, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.5333333333333333, "Spanish,English": 0.6, "Malay,Indonesian": 0.6866666666666666, "Malay,English": 0.5066666666666667, "Indonesian,English": 0.5133333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.41333333333333333, "Filipino,Vietnamese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Malay": 0.42, "Filipino,Vietnamese,Indonesian": 0.38666666666666666, "Filipino,Vietnamese,English": 0.26, "Filipino,Chinese,Spanish": 0.28, "Filipino,Chinese,Malay": 0.44, "Filipino,Chinese,Indonesian": 0.4066666666666667, "Filipino,Chinese,English": 0.28, "Filipino,Spanish,Malay": 0.36666666666666664, "Filipino,Spanish,Indonesian": 0.3466666666666667, "Filipino,Spanish,English": 0.36, "Filipino,Malay,Indonesian": 0.5, "Filipino,Malay,English": 0.36666666666666664, "Filipino,Indonesian,English": 0.35333333333333333, "Vietnamese,Chinese,Spanish": 0.31333333333333335, "Vietnamese,Chinese,Malay": 0.44666666666666666, "Vietnamese,Chinese,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,English": 0.26666666666666666, "Vietnamese,Spanish,Malay": 0.3, "Vietnamese,Spanish,Indonesian": 0.32, "Vietnamese,Spanish,English": 0.26, "Vietnamese,Malay,Indonesian": 0.43333333333333335, "Vietnamese,Malay,English": 0.2733333333333333, "Vietnamese,Indonesian,English": 0.29333333333333333, "Chinese,Spanish,Malay": 0.29333333333333333, "Chinese,Spanish,Indonesian": 0.32, "Chinese,Spanish,English": 0.2733333333333333, "Chinese,Malay,Indonesian": 0.4666666666666667, "Chinese,Malay,English": 0.2866666666666667, "Chinese,Indonesian,English": 0.31333333333333335, "Spanish,Malay,Indonesian": 0.4, "Spanish,Malay,English": 0.36666666666666664, "Spanish,Indonesian,English": 0.37333333333333335, "Malay,Indonesian,English": 0.4 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.34, "Filipino,Vietnamese,Chinese,Indonesian": 0.32, "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,English": 0.2, "Filipino,Vietnamese,Malay,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.23333333333333334, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, "Filipino,Chinese,Malay,English": 0.24, "Filipino,Chinese,Indonesian,English": 0.24666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.3, "Filipino,Spanish,Malay,English": 0.2733333333333333, "Filipino,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Malay,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.36, "Vietnamese,Chinese,Malay,English": 0.21333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.24666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.22, "Vietnamese,Malay,Indonesian,English": 0.25333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, "Chinese,Spanish,Malay,English": 0.21333333333333335, "Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Chinese,Malay,Indonesian,English": 0.26, "Spanish,Malay,Indonesian,English": 0.30666666666666664 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.18, "Filipino,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, "Vietnamese,Chinese,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, "AC3_2": 0.42917111343465364, "AC3_3": 0.35568304960895775, "AC3_4": 0.2987614326533628, "AC3_5": 0.2532035870560433, "AC3_6": 0.21489467158135575, "AC3_7": 0.17988071566821576 }, "prompt_5": { "overall_acc": 0.36380952380952386, "language_acc": { "Filipino": 0.30666666666666664, "Vietnamese": 0.26, "Chinese": 0.26, "Spanish": 0.4533333333333333, "Malay": 0.35333333333333333, "Indonesian": 0.37333333333333335, "English": 0.54 }, "consistency_score_2": 0.48571428571428565, "consistency_score_3": 0.29657142857142854, "consistency_score_4": 0.19504761904761914, "consistency_score_5": 0.13238095238095238, "consistency_score_6": 0.09238095238095237, "consistency_score_7": 0.06666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5266666666666666, "Filipino,Chinese": 0.4, "Filipino,Spanish": 0.5066666666666667, "Filipino,Malay": 0.62, "Filipino,Indonesian": 0.56, "Filipino,English": 0.5, "Vietnamese,Chinese": 0.47333333333333333, "Vietnamese,Spanish": 0.43333333333333335, "Vietnamese,Malay": 0.5133333333333333, "Vietnamese,Indonesian": 0.52, "Vietnamese,English": 0.3466666666666667, "Chinese,Spanish": 0.34, "Chinese,Malay": 0.42, "Chinese,Indonesian": 0.4866666666666667, "Chinese,English": 0.31333333333333335, "Spanish,Malay": 0.49333333333333335, "Spanish,Indonesian": 0.5133333333333333, "Spanish,English": 0.6, "Malay,Indonesian": 0.6733333333333333, "Malay,English": 0.47333333333333333, "Indonesian,English": 0.4866666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2733333333333333, "Filipino,Vietnamese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Malay": 0.38, "Filipino,Vietnamese,Indonesian": 0.36, "Filipino,Vietnamese,English": 0.25333333333333335, "Filipino,Chinese,Spanish": 0.19333333333333333, "Filipino,Chinese,Malay": 0.28, "Filipino,Chinese,Indonesian": 0.29333333333333333, "Filipino,Chinese,English": 0.21333333333333335, "Filipino,Spanish,Malay": 0.37333333333333335, "Filipino,Spanish,Indonesian": 0.32666666666666666, "Filipino,Spanish,English": 0.36666666666666664, "Filipino,Malay,Indonesian": 0.47333333333333333, "Filipino,Malay,English": 0.3466666666666667, "Filipino,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish": 0.22, "Vietnamese,Chinese,Malay": 0.26, "Vietnamese,Chinese,Indonesian": 0.30666666666666664, "Vietnamese,Chinese,English": 0.18, "Vietnamese,Spanish,Malay": 0.3, "Vietnamese,Spanish,Indonesian": 0.30666666666666664, "Vietnamese,Spanish,English": 0.26, "Vietnamese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Malay,English": 0.24, "Vietnamese,Indonesian,English": 0.25333333333333335, "Chinese,Spanish,Malay": 0.20666666666666667, "Chinese,Spanish,Indonesian": 0.23333333333333334, "Chinese,Spanish,English": 0.22, "Chinese,Malay,Indonesian": 0.3333333333333333, "Chinese,Malay,English": 0.20666666666666667, "Chinese,Indonesian,English": 0.25333333333333335, "Spanish,Malay,Indonesian": 0.38666666666666666, "Spanish,Malay,English": 0.36, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.36 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,English": 0.14, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,English": 0.2, "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay": 0.14, "Filipino,Chinese,Spanish,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Chinese,Malay,English": 0.16, "Filipino,Chinese,Indonesian,English": 0.18, "Filipino,Spanish,Malay,Indonesian": 0.3, "Filipino,Spanish,Malay,English": 0.2733333333333333, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, "Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Malay,English": 0.12, "Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.2, "Vietnamese,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Chinese,Spanish,Malay,English": 0.15333333333333332, "Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Chinese,Malay,Indonesian,English": 0.17333333333333334, "Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,English": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.1, "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 } }, "AC3_2": 0.4160153747107989, "AC3_3": 0.326767481099309, "AC3_4": 0.2539474794235115, "AC3_5": 0.19412485143697603, "AC3_6": 0.14734665470475203, "AC3_7": 0.11268436575553424 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.34983766233766234, "language_acc": { "Indonesian": 0.35795454545454547, "English": 0.5284090909090909, "Filipino": 0.30113636363636365, "Spanish": 0.42045454545454547, "Chinese": 0.24431818181818182, "Malay": 0.3352272727272727, "Vietnamese": 0.26136363636363635 }, "consistency_score_2": 0.417478354978355, "consistency_score_3": 0.21948051948051953, "consistency_score_4": 0.13441558441558446, "consistency_score_5": 0.09117965367965367, "consistency_score_6": 0.06655844155844155, "consistency_score_7": 0.05113636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3806818181818182, "Indonesian,Filipino": 0.4659090909090909, "Indonesian,Spanish": 0.4602272727272727, "Indonesian,Chinese": 0.39204545454545453, "Indonesian,Malay": 0.5568181818181818, "Indonesian,Vietnamese": 0.42613636363636365, "English,Filipino": 0.3409090909090909, "English,Spanish": 0.6079545454545454, "English,Chinese": 0.2840909090909091, "English,Malay": 0.3352272727272727, "English,Vietnamese": 0.29545454545454547, "Filipino,Spanish": 0.4090909090909091, "Filipino,Chinese": 0.39204545454545453, "Filipino,Malay": 0.4318181818181818, "Filipino,Vietnamese": 0.4431818181818182, "Spanish,Chinese": 0.29545454545454547, "Spanish,Malay": 0.4090909090909091, "Spanish,Vietnamese": 0.35795454545454547, "Chinese,Malay": 0.4943181818181818, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.4715909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.20454545454545456, "Indonesian,English,Spanish": 0.2840909090909091, "Indonesian,English,Chinese": 0.17045454545454544, "Indonesian,English,Malay": 0.22727272727272727, "Indonesian,English,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish": 0.26136363636363635, "Indonesian,Filipino,Chinese": 0.23863636363636365, "Indonesian,Filipino,Malay": 0.2840909090909091, "Indonesian,Filipino,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Chinese": 0.1875, "Indonesian,Spanish,Malay": 0.26704545454545453, "Indonesian,Spanish,Vietnamese": 0.21022727272727273, "Indonesian,Chinese,Malay": 0.30113636363636365, "Indonesian,Chinese,Vietnamese": 0.25, "Indonesian,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish": 0.24431818181818182, "English,Filipino,Chinese": 0.14772727272727273, "English,Filipino,Malay": 0.17613636363636365, "English,Filipino,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese": 0.17613636363636365, "English,Spanish,Malay": 0.24431818181818182, "English,Spanish,Vietnamese": 0.2159090909090909, "English,Chinese,Malay": 0.1590909090909091, "English,Chinese,Vietnamese": 0.16477272727272727, "English,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese": 0.17045454545454544, "Filipino,Spanish,Malay": 0.2215909090909091, "Filipino,Spanish,Vietnamese": 0.19886363636363635, "Filipino,Chinese,Malay": 0.23863636363636365, "Filipino,Chinese,Vietnamese": 0.25, "Filipino,Malay,Vietnamese": 0.25, "Spanish,Chinese,Malay": 0.19318181818181818, "Spanish,Chinese,Vietnamese": 0.19318181818181818, "Spanish,Malay,Vietnamese": 0.2159090909090909, "Chinese,Malay,Vietnamese": 0.3125 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1534090909090909, "Indonesian,English,Filipino,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese": 0.125, "Indonesian,English,Spanish,Malay": 0.16477272727272727, "Indonesian,English,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,English,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Chinese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay": 0.17613636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,Spanish,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese": 0.11363636363636363, "English,Filipino,Spanish,Malay": 0.1534090909090909, "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay": 0.10227272727272728, "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, "English,Filipino,Malay,Vietnamese": 0.09659090909090909, "English,Spanish,Chinese,Malay": 0.11931818181818182, "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, "English,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Chinese,Malay,Vietnamese": 0.09659090909090909, "Filipino,Spanish,Chinese,Malay": 0.125, "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Malay,Vietnamese": 0.0625, "Indonesian,English,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Chinese,Malay,Vietnamese": 0.07954545454545454, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.0625, "English,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 } }, "AC3_2": 0.38067666637282366, "AC3_3": 0.26973511228824704, "AC3_4": 0.19421091810644825, "AC3_5": 0.14465680022215904, "AC3_6": 0.11183894075945042, "AC3_7": 0.08922984907601689 }, "prompt_2": { "overall_acc": 0.3538961038961039, "language_acc": { "Indonesian": 0.32386363636363635, "English": 0.5340909090909091, "Filipino": 0.3181818181818182, "Spanish": 0.4375, "Chinese": 0.24431818181818182, "Malay": 0.3522727272727273, "Vietnamese": 0.26704545454545453 }, "consistency_score_2": 0.42153679653679665, "consistency_score_3": 0.2202922077922078, "consistency_score_4": 0.13555194805194803, "consistency_score_5": 0.09361471861471862, "consistency_score_6": 0.0706168831168831, "consistency_score_7": 0.056818181818181816, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.38636363636363635, "Indonesian,Filipino": 0.48863636363636365, "Indonesian,Spanish": 0.44886363636363635, "Indonesian,Chinese": 0.3806818181818182, "Indonesian,Malay": 0.6022727272727273, "Indonesian,Vietnamese": 0.35795454545454547, "English,Filipino": 0.3465909090909091, "English,Spanish": 0.6306818181818182, "English,Chinese": 0.29545454545454547, "English,Malay": 0.3522727272727273, "English,Vietnamese": 0.29545454545454547, "Filipino,Spanish": 0.4602272727272727, "Filipino,Chinese": 0.42045454545454547, "Filipino,Malay": 0.4772727272727273, "Filipino,Vietnamese": 0.42613636363636365, "Spanish,Chinese": 0.29545454545454547, "Spanish,Malay": 0.3977272727272727, "Spanish,Vietnamese": 0.3522727272727273, "Chinese,Malay": 0.4772727272727273, "Chinese,Vietnamese": 0.5, "Malay,Vietnamese": 0.4602272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.22727272727272727, "Indonesian,English,Spanish": 0.2897727272727273, "Indonesian,English,Chinese": 0.18181818181818182, "Indonesian,English,Malay": 0.2556818181818182, "Indonesian,English,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish": 0.2727272727272727, "Indonesian,Filipino,Chinese": 0.23863636363636365, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Chinese": 0.18181818181818182, "Indonesian,Spanish,Malay": 0.2897727272727273, "Indonesian,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,Chinese,Malay": 0.2897727272727273, "Indonesian,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish": 0.2840909090909091, "English,Filipino,Chinese": 0.1590909090909091, "English,Filipino,Malay": 0.1875, "English,Filipino,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese": 0.19886363636363635, "English,Spanish,Malay": 0.2556818181818182, "English,Spanish,Vietnamese": 0.2159090909090909, "English,Chinese,Malay": 0.17045454545454544, "English,Chinese,Vietnamese": 0.1534090909090909, "English,Malay,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Chinese": 0.19886363636363635, "Filipino,Spanish,Malay": 0.25, "Filipino,Spanish,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay": 0.26704545454545453, "Filipino,Chinese,Vietnamese": 0.23863636363636365, "Filipino,Malay,Vietnamese": 0.24431818181818182, "Spanish,Chinese,Malay": 0.1875, "Spanish,Chinese,Vietnamese": 0.17045454545454544, "Spanish,Malay,Vietnamese": 0.1875, "Chinese,Malay,Vietnamese": 0.2897727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.18181818181818182, "Indonesian,English,Filipino,Chinese": 0.125, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese": 0.13636363636363635, "Indonesian,English,Spanish,Malay": 0.19318181818181818, "Indonesian,English,Spanish,Vietnamese": 0.11931818181818182, "Indonesian,English,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,English,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Malay": 0.19318181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay": 0.1875, "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Chinese,Malay,Vietnamese": 0.1875, "English,Filipino,Spanish,Chinese": 0.13636363636363635, "English,Filipino,Spanish,Malay": 0.16477272727272727, "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, "English,Filipino,Chinese,Malay": 0.125, "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, "English,Filipino,Malay,Vietnamese": 0.09090909090909091, "English,Spanish,Chinese,Malay": 0.13636363636363635, "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, "English,Spanish,Malay,Vietnamese": 0.11363636363636363, "English,Chinese,Malay,Vietnamese": 0.09659090909090909, "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.125, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0625, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 } }, "AC3_2": 0.38476631528194355, "AC3_3": 0.27155047379325725, "AC3_4": 0.19602205419095983, "AC3_5": 0.14806294065379674, "AC3_6": 0.11773981274075816, "AC3_7": 0.09791591805018045 }, "prompt_3": { "overall_acc": 0.3547077922077922, "language_acc": { "Indonesian": 0.3409090909090909, "English": 0.5227272727272727, "Filipino": 0.3125, "Spanish": 0.4375, "Chinese": 0.25, "Malay": 0.3522727272727273, "Vietnamese": 0.26704545454545453 }, "consistency_score_2": 0.4315476190476191, "consistency_score_3": 0.23344155844155842, "consistency_score_4": 0.15162337662337666, "consistency_score_5": 0.1130952380952381, "consistency_score_6": 0.09253246753246754, "consistency_score_7": 0.07954545454545454, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.39204545454545453, "Indonesian,Filipino": 0.4943181818181818, "Indonesian,Spanish": 0.4431818181818182, "Indonesian,Chinese": 0.38636363636363635, "Indonesian,Malay": 0.5965909090909091, "Indonesian,Vietnamese": 0.38636363636363635, "English,Filipino": 0.3352272727272727, "English,Spanish": 0.6420454545454546, "English,Chinese": 0.32954545454545453, "English,Malay": 0.375, "English,Vietnamese": 0.3125, "Filipino,Spanish": 0.4375, "Filipino,Chinese": 0.4034090909090909, "Filipino,Malay": 0.4772727272727273, "Filipino,Vietnamese": 0.4715909090909091, "Spanish,Chinese": 0.3125, "Spanish,Malay": 0.4375, "Spanish,Vietnamese": 0.36363636363636365, "Chinese,Malay": 0.48295454545454547, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.4659090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.22727272727272727, "Indonesian,English,Spanish": 0.29545454545454547, "Indonesian,English,Chinese": 0.19318181818181818, "Indonesian,English,Malay": 0.26136363636363635, "Indonesian,English,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish": 0.2727272727272727, "Indonesian,Filipino,Chinese": 0.23863636363636365, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.19886363636363635, "Indonesian,Spanish,Malay": 0.29545454545454547, "Indonesian,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Chinese,Malay": 0.3068181818181818, "Indonesian,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish": 0.26704545454545453, "English,Filipino,Chinese": 0.16477272727272727, "English,Filipino,Malay": 0.19886363636363635, "English,Filipino,Vietnamese": 0.1534090909090909, "English,Spanish,Chinese": 0.2215909090909091, "English,Spanish,Malay": 0.2784090909090909, "English,Spanish,Vietnamese": 0.2215909090909091, "English,Chinese,Malay": 0.19318181818181818, "English,Chinese,Vietnamese": 0.17613636363636365, "English,Malay,Vietnamese": 0.16477272727272727, "Filipino,Spanish,Chinese": 0.19318181818181818, "Filipino,Spanish,Malay": 0.26136363636363635, "Filipino,Spanish,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay": 0.26704545454545453, "Filipino,Chinese,Vietnamese": 0.26704545454545453, "Filipino,Malay,Vietnamese": 0.2727272727272727, "Spanish,Chinese,Malay": 0.20454545454545456, "Spanish,Chinese,Vietnamese": 0.19318181818181818, "Spanish,Malay,Vietnamese": 0.2159090909090909, "Chinese,Malay,Vietnamese": 0.3125 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.18181818181818182, "Indonesian,English,Filipino,Chinese": 0.13068181818181818, "Indonesian,English,Filipino,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Vietnamese": 0.11363636363636363, "Indonesian,English,Spanish,Chinese": 0.14772727272727273, "Indonesian,English,Spanish,Malay": 0.19886363636363635, "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,English,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Malay": 0.20454545454545456, "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Chinese,Malay": 0.19886363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.19318181818181818, "English,Filipino,Spanish,Chinese": 0.14204545454545456, "English,Filipino,Spanish,Malay": 0.17613636363636365, "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay": 0.13636363636363635, "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.14772727272727273, "English,Spanish,Chinese,Vietnamese": 0.13636363636363635, "English,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Chinese,Malay,Vietnamese": 0.11931818181818182, "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay": 0.125, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Filipino,Spanish,Chinese,Malay": 0.125, "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 } }, "AC3_2": 0.38937297212627153, "AC3_3": 0.2815731742247202, "AC3_4": 0.2124380108090074, "AC3_5": 0.17150706432754498, "AC3_6": 0.1467756381221304, "AC3_7": 0.12994902290975244 }, "prompt_4": { "overall_acc": 0.35227272727272724, "language_acc": { "Indonesian": 0.3352272727272727, "English": 0.5340909090909091, "Filipino": 0.3068181818181818, "Spanish": 0.42613636363636365, "Chinese": 0.24431818181818182, "Malay": 0.3465909090909091, "Vietnamese": 0.2727272727272727 }, "consistency_score_2": 0.43181818181818166, "consistency_score_3": 0.23522727272727276, "consistency_score_4": 0.1525974025974026, "consistency_score_5": 0.11282467532467534, "consistency_score_6": 0.09172077922077922, "consistency_score_7": 0.07954545454545454, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.38636363636363635, "Indonesian,Filipino": 0.5, "Indonesian,Spanish": 0.45454545454545453, "Indonesian,Chinese": 0.3806818181818182, "Indonesian,Malay": 0.5909090909090909, "Indonesian,Vietnamese": 0.39204545454545453, "English,Filipino": 0.3409090909090909, "English,Spanish": 0.6363636363636364, "English,Chinese": 0.30113636363636365, "English,Malay": 0.375, "English,Vietnamese": 0.3068181818181818, "Filipino,Spanish": 0.4602272727272727, "Filipino,Chinese": 0.42045454545454547, "Filipino,Malay": 0.4659090909090909, "Filipino,Vietnamese": 0.4602272727272727, "Spanish,Chinese": 0.3068181818181818, "Spanish,Malay": 0.4147727272727273, "Spanish,Vietnamese": 0.36363636363636365, "Chinese,Malay": 0.5, "Chinese,Vietnamese": 0.5340909090909091, "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.22727272727272727, "Indonesian,English,Spanish": 0.2897727272727273, "Indonesian,English,Chinese": 0.18181818181818182, "Indonesian,English,Malay": 0.25, "Indonesian,English,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish": 0.2840909090909091, "Indonesian,Filipino,Chinese": 0.25, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.19318181818181818, "Indonesian,Spanish,Malay": 0.2897727272727273, "Indonesian,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Chinese,Malay": 0.30113636363636365, "Indonesian,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish": 0.2727272727272727, "English,Filipino,Chinese": 0.16477272727272727, "English,Filipino,Malay": 0.19318181818181818, "English,Filipino,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese": 0.2159090909090909, "English,Spanish,Malay": 0.2727272727272727, "English,Spanish,Vietnamese": 0.22727272727272727, "English,Chinese,Malay": 0.19318181818181818, "English,Chinese,Vietnamese": 0.17045454545454544, "English,Malay,Vietnamese": 0.17613636363636365, "Filipino,Spanish,Chinese": 0.20454545454545456, "Filipino,Spanish,Malay": 0.2556818181818182, "Filipino,Spanish,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay": 0.2784090909090909, "Filipino,Chinese,Vietnamese": 0.2727272727272727, "Filipino,Malay,Vietnamese": 0.2727272727272727, "Spanish,Chinese,Malay": 0.19886363636363635, "Spanish,Chinese,Vietnamese": 0.19886363636363635, "Spanish,Malay,Vietnamese": 0.2215909090909091, "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.18181818181818182, "Indonesian,English,Filipino,Chinese": 0.13068181818181818, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.11363636363636363, "Indonesian,English,Spanish,Chinese": 0.14772727272727273, "Indonesian,English,Spanish,Malay": 0.19318181818181818, "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,English,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Chinese,Malay": 0.20454545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.1875, "Indonesian,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese": 0.14772727272727273, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay": 0.13068181818181818, "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.14772727272727273, "English,Spanish,Chinese,Vietnamese": 0.13636363636363635, "English,Spanish,Malay,Vietnamese": 0.14204545454545456, "English,Chinese,Malay,Vietnamese": 0.125, "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay": 0.11931818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.125, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 } }, "AC3_2": 0.3880105401349677, "AC3_3": 0.2820907332032594, "AC3_4": 0.21294942994905158, "AC3_5": 0.1709106774184328, "AC3_6": 0.14554595310000934, "AC3_7": 0.1297846889651599 }, "prompt_5": { "overall_acc": 0.3538961038961039, "language_acc": { "Indonesian": 0.32954545454545453, "English": 0.5284090909090909, "Filipino": 0.3125, "Spanish": 0.4318181818181818, "Chinese": 0.26136363636363635, "Malay": 0.3522727272727273, "Vietnamese": 0.26136363636363635 }, "consistency_score_2": 0.4218073593073593, "consistency_score_3": 0.22532467532467537, "consistency_score_4": 0.1430194805194805, "consistency_score_5": 0.10173160173160173, "consistency_score_6": 0.07792207792207792, "consistency_score_7": 0.0625, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.39204545454545453, "Indonesian,Filipino": 0.4602272727272727, "Indonesian,Spanish": 0.4431818181818182, "Indonesian,Chinese": 0.36363636363636365, "Indonesian,Malay": 0.5795454545454546, "Indonesian,Vietnamese": 0.42045454545454547, "English,Filipino": 0.32386363636363635, "English,Spanish": 0.6363636363636364, "English,Chinese": 0.32954545454545453, "English,Malay": 0.36363636363636365, "English,Vietnamese": 0.3068181818181818, "Filipino,Spanish": 0.4431818181818182, "Filipino,Chinese": 0.38636363636363635, "Filipino,Malay": 0.5, "Filipino,Vietnamese": 0.4147727272727273, "Spanish,Chinese": 0.32954545454545453, "Spanish,Malay": 0.4034090909090909, "Spanish,Vietnamese": 0.3352272727272727, "Chinese,Malay": 0.48863636363636365, "Chinese,Vietnamese": 0.4602272727272727, "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.21022727272727273, "Indonesian,English,Spanish": 0.3068181818181818, "Indonesian,English,Chinese": 0.18181818181818182, "Indonesian,English,Malay": 0.2556818181818182, "Indonesian,English,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish": 0.2556818181818182, "Indonesian,Filipino,Chinese": 0.2215909090909091, "Indonesian,Filipino,Malay": 0.3181818181818182, "Indonesian,Filipino,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Chinese": 0.19318181818181818, "Indonesian,Spanish,Malay": 0.2840909090909091, "Indonesian,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,Chinese,Malay": 0.2897727272727273, "Indonesian,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,Malay,Vietnamese": 0.2840909090909091, "English,Filipino,Spanish": 0.26136363636363635, "English,Filipino,Chinese": 0.16477272727272727, "English,Filipino,Malay": 0.20454545454545456, "English,Filipino,Vietnamese": 0.13636363636363635, "English,Spanish,Chinese": 0.23295454545454544, "English,Spanish,Malay": 0.2840909090909091, "English,Spanish,Vietnamese": 0.22727272727272727, "English,Chinese,Malay": 0.19318181818181818, "English,Chinese,Vietnamese": 0.1590909090909091, "English,Malay,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Chinese": 0.20454545454545456, "Filipino,Spanish,Malay": 0.2556818181818182, "Filipino,Spanish,Vietnamese": 0.1875, "Filipino,Chinese,Malay": 0.2727272727272727, "Filipino,Chinese,Vietnamese": 0.21022727272727273, "Filipino,Malay,Vietnamese": 0.26136363636363635, "Spanish,Chinese,Malay": 0.20454545454545456, "Spanish,Chinese,Vietnamese": 0.16477272727272727, "Spanish,Malay,Vietnamese": 0.20454545454545456, "Chinese,Malay,Vietnamese": 0.2840909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.17045454545454544, "Indonesian,English,Filipino,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese": 0.1534090909090909, "Indonesian,English,Spanish,Malay": 0.21022727272727273, "Indonesian,English,Spanish,Vietnamese": 0.14772727272727273, "Indonesian,English,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Chinese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay": 0.18181818181818182, "Indonesian,Filipino,Chinese,Vietnamese": 0.125, "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Chinese,Malay,Vietnamese": 0.1875, "English,Filipino,Spanish,Chinese": 0.1534090909090909, "English,Filipino,Spanish,Malay": 0.17613636363636365, "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, "English,Filipino,Chinese,Malay": 0.13068181818181818, "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.1534090909090909, "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Spanish,Malay,Vietnamese": 0.14204545454545456, "English,Chinese,Malay,Vietnamese": 0.11363636363636363, "Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.10795454545454546, "Indonesian,English,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, "English,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 } }, "AC3_2": 0.3848789856316012, "AC3_3": 0.2753406906518987, "AC3_4": 0.20371281768685, "AC3_5": 0.15803436465051104, "AC3_6": 0.12772190212841467, "AC3_7": 0.1062378167386189 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6116504854368932 }, "prompt_2": { "accuracy": 0.6310679611650486 }, "prompt_3": { "accuracy": 0.5533980582524272 }, "prompt_4": { "accuracy": 0.6019417475728155 }, "prompt_5": { "accuracy": 0.5533980582524272 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2571428571428571 }, "prompt_2": { "accuracy": 0.29523809523809524 }, "prompt_3": { "accuracy": 0.3047619047619048 }, "prompt_4": { "accuracy": 0.2761904761904762 }, "prompt_5": { "accuracy": 0.2857142857142857 } }, "us_eval": { "prompt_1": { "accuracy": 0.5607476635514018 }, "prompt_2": { "accuracy": 0.5233644859813084 }, "prompt_3": { "accuracy": 0.5794392523364486 }, "prompt_4": { "accuracy": 0.5607476635514018 }, "prompt_5": { "accuracy": 0.5794392523364486 } }, "ph_eval": { "prompt_1": { "accuracy": 0.3, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.4, "politics": 0.2, "culture": 0.3, "film": 0.1, "law": 0.3, "geography": 0.5 } }, "prompt_2": { "accuracy": 0.28, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.4, "politics": 0.2, "culture": 0.3, "film": 0.1, "law": 0.2, "geography": 0.5 } }, "prompt_3": { "accuracy": 0.32, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.3333333333333333, "literature": 0.4, "politics": 0.2, "culture": 0.4, "film": 0.2, "law": 0.3, "geography": 0.4 } }, "prompt_4": { "accuracy": 0.28, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.2, "literature": 0.5, "politics": 0.2, "culture": 0.3, "film": 0.1, "law": 0.2, "geography": 0.4 } }, "prompt_5": { "accuracy": 0.32, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.4, "politics": 0.2, "culture": 0.4, "film": 0.3, "law": 0.3, "geography": 0.4 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.1757121671640427 }, "prompt_2": { "bleu_score": 0.1713078295205753 }, "prompt_3": { "bleu_score": 0.1784941784513121 }, "prompt_4": { "bleu_score": 0.18529617182140173 }, "prompt_5": { "bleu_score": 0.16808729137211453 } }, "indommlu": { "prompt_1": { "accuracy": 0.29601442018826357, "category_acc": { "History": 0.285140562248996, "Geography": 0.21020408163265306, "Lampungic": 0.2789115646258503, "Social science": 0.3639398998330551, "Balinese": 0.3184713375796178, "Makassarese": 0.24193548387096775, "Banjarese": 0.2361111111111111, "Chemistry": 0.21897810218978103, "Biology": 0.2828402366863905, "Science": 0.3013415892672859, "Christian religion": 0.3781094527363184, "Art": 0.2895174708818636, "Islam religion": 0.310099573257468, "Hindu religion": 0.29333333333333333, "Madurese": 0.29491525423728815, "Sport": 0.32432432432432434, "Indonesian language": 0.3297011207970112, "Physics": 0.26464646464646463, "Minangkabau culture": 0.271356783919598, "Dayak language": 0.30275229357798167, "Sociology": 0.28830645161290325, "Economy": 0.2848360655737705, "Sundanese": 0.28867761452031115, "Javanese": 0.2590725806451613, "Civic education": 0.3190271816881259 } }, "prompt_2": { "accuracy": 0.2902730489351759, "category_acc": { "History": 0.29116465863453816, "Geography": 0.21224489795918366, "Lampungic": 0.32653061224489793, "Social science": 0.34557595993322204, "Balinese": 0.3099787685774947, "Makassarese": 0.24193548387096775, "Banjarese": 0.2152777777777778, "Chemistry": 0.23357664233576642, "Biology": 0.27928994082840236, "Science": 0.2899896800825593, "Christian religion": 0.3383084577114428, "Art": 0.26788685524126454, "Islam religion": 0.3157894736842105, "Hindu religion": 0.2733333333333333, "Madurese": 0.2847457627118644, "Sport": 0.3108108108108108, "Indonesian language": 0.31787048567870485, "Physics": 0.25656565656565655, "Minangkabau culture": 0.271356783919598, "Dayak language": 0.30275229357798167, "Sociology": 0.26814516129032256, "Economy": 0.26024590163934425, "Sundanese": 0.29472774416594644, "Javanese": 0.26310483870967744, "Civic education": 0.32331902718168815 } }, "prompt_3": { "accuracy": 0.2936778156085186, "category_acc": { "History": 0.30120481927710846, "Geography": 0.20408163265306123, "Lampungic": 0.3129251700680272, "Social science": 0.3606010016694491, "Balinese": 0.3099787685774947, "Makassarese": 0.20967741935483872, "Banjarese": 0.2222222222222222, "Chemistry": 0.22481751824817517, "Biology": 0.2781065088757396, "Science": 0.30030959752321984, "Christian religion": 0.3383084577114428, "Art": 0.26955074875207985, "Islam religion": 0.29445234708392604, "Hindu religion": 0.2733333333333333, "Madurese": 0.2677966101694915, "Sport": 0.3108108108108108, "Indonesian language": 0.3234744707347447, "Physics": 0.25252525252525254, "Minangkabau culture": 0.2864321608040201, "Dayak language": 0.30275229357798167, "Sociology": 0.3004032258064516, "Economy": 0.27049180327868855, "Sundanese": 0.31028522039757994, "Javanese": 0.2782258064516129, "Civic education": 0.3104434907010014 } }, "prompt_4": { "accuracy": 0.29127445089792375, "category_acc": { "History": 0.27710843373493976, "Geography": 0.23061224489795917, "Lampungic": 0.29931972789115646, "Social science": 0.35392320534223703, "Balinese": 0.3248407643312102, "Makassarese": 0.26344086021505375, "Banjarese": 0.22916666666666666, "Chemistry": 0.21313868613138687, "Biology": 0.28402366863905326, "Science": 0.2786377708978328, "Christian religion": 0.373134328358209, "Art": 0.2995008319467554, "Islam religion": 0.29445234708392604, "Hindu religion": 0.28, "Madurese": 0.288135593220339, "Sport": 0.2972972972972973, "Indonesian language": 0.31880448318804483, "Physics": 0.25656565656565655, "Minangkabau culture": 0.27638190954773867, "Dayak language": 0.29357798165137616, "Sociology": 0.29435483870967744, "Economy": 0.2786885245901639, "Sundanese": 0.2878133102852204, "Javanese": 0.2610887096774194, "Civic education": 0.3147353361945637 } }, "prompt_5": { "accuracy": 0.2872688430469324, "category_acc": { "History": 0.27309236947791166, "Geography": 0.20204081632653062, "Lampungic": 0.30612244897959184, "Social science": 0.34390651085141904, "Balinese": 0.3036093418259023, "Makassarese": 0.27419354838709675, "Banjarese": 0.2013888888888889, "Chemistry": 0.22043795620437956, "Biology": 0.263905325443787, "Science": 0.2786377708978328, "Christian religion": 0.34328358208955223, "Art": 0.28452579034941766, "Islam religion": 0.2958748221906117, "Hindu religion": 0.29333333333333333, "Madurese": 0.2983050847457627, "Sport": 0.30405405405405406, "Indonesian language": 0.3234744707347447, "Physics": 0.23636363636363636, "Minangkabau culture": 0.2663316582914573, "Dayak language": 0.25688073394495414, "Sociology": 0.28830645161290325, "Economy": 0.2725409836065574, "Sundanese": 0.2938634399308557, "Javanese": 0.2540322580645161, "Civic education": 0.3147353361945637 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.23519019047974593 }, "prompt_2": { "bleu_score": 0.23081721810773512 }, "prompt_3": { "bleu_score": 0.2317036111734395 }, "prompt_4": { "bleu_score": 0.23574032794862365 }, "prompt_5": { "bleu_score": 0.22734515454295717 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.05726857609979452 }, "prompt_2": { "bleu_score": 0.056030313148321104 }, "prompt_3": { "bleu_score": 0.056220586684100834 }, "prompt_4": { "bleu_score": 0.05707255304827606 }, "prompt_5": { "bleu_score": 0.05453166476110994 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.01556584838007841 }, "prompt_2": { "bleu_score": 0.016250348764526006 }, "prompt_3": { "bleu_score": 0.015726636553525738 }, "prompt_4": { "bleu_score": 0.018325681250757293 }, "prompt_5": { "bleu_score": 0.014647484001582058 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.21964506743805942 }, "prompt_2": { "bleu_score": 0.21688082199907668 }, "prompt_3": { "bleu_score": 0.21728844545846515 }, "prompt_4": { "bleu_score": 0.22147378722039016 }, "prompt_5": { "bleu_score": 0.20762462597989922 } }, "mmlu": { "prompt_1": { "accuracy": 0.5029171528588098 }, "prompt_2": { "accuracy": 0.5040840140023337 }, "prompt_3": { "accuracy": 0.5122520420070011 }, "prompt_4": { "accuracy": 0.5099183197199533 }, "prompt_5": { "accuracy": 0.5134189031505251 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.4889524490525563, "category_acc": { "high_school_european_history": 0.7012195121951219, "business_ethics": 0.6464646464646465, "clinical_knowledge": 0.5606060606060606, "medical_genetics": 0.46464646464646464, "high_school_us_history": 0.6847290640394089, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.6694915254237288, "virology": 0.43636363636363634, "high_school_microeconomics": 0.510548523206751, "econometrics": 0.30973451327433627, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.5792880258899676, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.3736654804270463, "philosophy": 0.5225806451612903, "professional_medicine": 0.4575645756457565, "nutrition": 0.4786885245901639, "global_facts": 0.31313131313131315, "machine_learning": 0.35135135135135137, "security_studies": 0.5737704918032787, "public_relations": 0.5871559633027523, "professional_psychology": 0.4795417348608838, "prehistory": 0.47678018575851394, "anatomy": 0.4552238805970149, "human_sexuality": 0.5769230769230769, "college_medicine": 0.47093023255813954, "high_school_government_and_politics": 0.6875, "college_chemistry": 0.40404040404040403, "logical_fallacies": 0.5555555555555556, "high_school_geography": 0.6700507614213198, "elementary_mathematics": 0.34748010610079577, "human_aging": 0.536036036036036, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.6911764705882353, "formal_logic": 0.352, "high_school_statistics": 0.4046511627906977, "international_law": 0.65, "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.494949494949495, "conceptual_physics": 0.4444444444444444, "miscellaneous": 0.659846547314578, "high_school_chemistry": 0.30198019801980197, "marketing": 0.7811158798283262, "professional_law": 0.3842139595564253, "management": 0.6764705882352942, "college_physics": 0.3465346534653465, "jurisprudence": 0.5794392523364486, "world_religions": 0.5764705882352941, "sociology": 0.73, "us_foreign_policy": 0.7272727272727273, "high_school_macroeconomics": 0.5244215938303342, "computer_security": 0.5858585858585859, "moral_scenarios": 0.23154362416107382, "moral_disputes": 0.5797101449275363, "electrical_engineering": 0.4027777777777778, "astronomy": 0.5099337748344371, "college_biology": 0.45454545454545453 } }, "prompt_2": { "accuracy": 0.48859492313192704, "category_acc": { "high_school_european_history": 0.7073170731707317, "business_ethics": 0.6262626262626263, "clinical_knowledge": 0.5681818181818182, "medical_genetics": 0.494949494949495, "high_school_us_history": 0.6847290640394089, "high_school_physics": 0.3, "high_school_world_history": 0.6864406779661016, "virology": 0.4303030303030303, "high_school_microeconomics": 0.510548523206751, "econometrics": 0.336283185840708, "college_computer_science": 0.3939393939393939, "high_school_biology": 0.56957928802589, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.37722419928825623, "philosophy": 0.5225806451612903, "professional_medicine": 0.44649446494464945, "nutrition": 0.4721311475409836, "global_facts": 0.3333333333333333, "machine_learning": 0.36036036036036034, "security_studies": 0.5860655737704918, "public_relations": 0.5688073394495413, "professional_psychology": 0.46317512274959083, "prehistory": 0.49226006191950467, "anatomy": 0.44029850746268656, "human_sexuality": 0.5538461538461539, "college_medicine": 0.48255813953488375, "high_school_government_and_politics": 0.6822916666666666, "college_chemistry": 0.43434343434343436, "logical_fallacies": 0.5802469135802469, "high_school_geography": 0.6700507614213198, "elementary_mathematics": 0.3209549071618037, "human_aging": 0.5135135135135135, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.6930147058823529, "formal_logic": 0.392, "high_school_statistics": 0.4186046511627907, "international_law": 0.65, "high_school_mathematics": 0.3048327137546468, "high_school_computer_science": 0.46464646464646464, "conceptual_physics": 0.42735042735042733, "miscellaneous": 0.6483375959079284, "high_school_chemistry": 0.3069306930693069, "marketing": 0.7467811158798283, "professional_law": 0.38290932811480755, "management": 0.6764705882352942, "college_physics": 0.31683168316831684, "jurisprudence": 0.5887850467289719, "world_religions": 0.5941176470588235, "sociology": 0.74, "us_foreign_policy": 0.7171717171717171, "high_school_macroeconomics": 0.5244215938303342, "computer_security": 0.5959595959595959, "moral_scenarios": 0.2203579418344519, "moral_disputes": 0.5855072463768116, "electrical_engineering": 0.4166666666666667, "astronomy": 0.543046357615894, "college_biology": 0.4965034965034965 } }, "prompt_3": { "accuracy": 0.488165892027172, "category_acc": { "high_school_european_history": 0.7073170731707317, "business_ethics": 0.6161616161616161, "clinical_knowledge": 0.5454545454545454, "medical_genetics": 0.494949494949495, "high_school_us_history": 0.6847290640394089, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.6694915254237288, "virology": 0.44242424242424244, "high_school_microeconomics": 0.5147679324894515, "econometrics": 0.336283185840708, "college_computer_science": 0.41414141414141414, "high_school_biology": 0.56957928802589, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.3879003558718861, "philosophy": 0.5225806451612903, "professional_medicine": 0.45018450184501846, "nutrition": 0.4819672131147541, "global_facts": 0.3434343434343434, "machine_learning": 0.32432432432432434, "security_studies": 0.5901639344262295, "public_relations": 0.5871559633027523, "professional_psychology": 0.4664484451718494, "prehistory": 0.47368421052631576, "anatomy": 0.44776119402985076, "human_sexuality": 0.5769230769230769, "college_medicine": 0.47674418604651164, "high_school_government_and_politics": 0.6822916666666666, "college_chemistry": 0.42424242424242425, "logical_fallacies": 0.5864197530864198, "high_school_geography": 0.6751269035532995, "elementary_mathematics": 0.33421750663129973, "human_aging": 0.5225225225225225, "college_mathematics": 0.24242424242424243, "high_school_psychology": 0.6911764705882353, "formal_logic": 0.376, "high_school_statistics": 0.413953488372093, "international_law": 0.65, "high_school_mathematics": 0.30111524163568776, "high_school_computer_science": 0.4444444444444444, "conceptual_physics": 0.43162393162393164, "miscellaneous": 0.6547314578005116, "high_school_chemistry": 0.30198019801980197, "marketing": 0.759656652360515, "professional_law": 0.3796477495107632, "management": 0.6764705882352942, "college_physics": 0.32673267326732675, "jurisprudence": 0.5887850467289719, "world_religions": 0.5764705882352941, "sociology": 0.735, "us_foreign_policy": 0.7070707070707071, "high_school_macroeconomics": 0.532133676092545, "computer_security": 0.6161616161616161, "moral_scenarios": 0.2225950782997763, "moral_disputes": 0.5855072463768116, "electrical_engineering": 0.4166666666666667, "astronomy": 0.5364238410596026, "college_biology": 0.46853146853146854 } }, "prompt_4": { "accuracy": 0.48623525205577406, "category_acc": { "high_school_european_history": 0.7073170731707317, "business_ethics": 0.6363636363636364, "clinical_knowledge": 0.553030303030303, "medical_genetics": 0.48484848484848486, "high_school_us_history": 0.6748768472906403, "high_school_physics": 0.3, "high_school_world_history": 0.6610169491525424, "virology": 0.4484848484848485, "high_school_microeconomics": 0.5189873417721519, "econometrics": 0.3274336283185841, "college_computer_science": 0.45454545454545453, "high_school_biology": 0.5631067961165048, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.3807829181494662, "philosophy": 0.5225806451612903, "professional_medicine": 0.44280442804428044, "nutrition": 0.47540983606557374, "global_facts": 0.3434343434343434, "machine_learning": 0.36036036036036034, "security_studies": 0.5778688524590164, "public_relations": 0.5779816513761468, "professional_psychology": 0.46153846153846156, "prehistory": 0.47368421052631576, "anatomy": 0.44029850746268656, "human_sexuality": 0.5461538461538461, "college_medicine": 0.48255813953488375, "high_school_government_and_politics": 0.6770833333333334, "college_chemistry": 0.42424242424242425, "logical_fallacies": 0.5555555555555556, "high_school_geography": 0.6802030456852792, "elementary_mathematics": 0.3183023872679045, "human_aging": 0.509009009009009, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.6875, "formal_logic": 0.368, "high_school_statistics": 0.413953488372093, "international_law": 0.6583333333333333, "high_school_mathematics": 0.2899628252788104, "high_school_computer_science": 0.47474747474747475, "conceptual_physics": 0.43162393162393164, "miscellaneous": 0.6483375959079284, "high_school_chemistry": 0.297029702970297, "marketing": 0.7424892703862661, "professional_law": 0.3822570123939987, "management": 0.6862745098039216, "college_physics": 0.3465346534653465, "jurisprudence": 0.5794392523364486, "world_religions": 0.5705882352941176, "sociology": 0.745, "us_foreign_policy": 0.6767676767676768, "high_school_macroeconomics": 0.5167095115681234, "computer_security": 0.6060606060606061, "moral_scenarios": 0.23154362416107382, "moral_disputes": 0.5826086956521739, "electrical_engineering": 0.4236111111111111, "astronomy": 0.5298013245033113, "college_biology": 0.4755244755244755 } }, "prompt_5": { "accuracy": 0.4868787987129067, "category_acc": { "high_school_european_history": 0.7134146341463414, "business_ethics": 0.6262626262626263, "clinical_knowledge": 0.5643939393939394, "medical_genetics": 0.5050505050505051, "high_school_us_history": 0.6650246305418719, "high_school_physics": 0.3, "high_school_world_history": 0.6779661016949152, "virology": 0.45454545454545453, "high_school_microeconomics": 0.5189873417721519, "econometrics": 0.3274336283185841, "college_computer_science": 0.42424242424242425, "high_school_biology": 0.5598705501618123, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.3665480427046263, "philosophy": 0.5, "professional_medicine": 0.45018450184501846, "nutrition": 0.4786885245901639, "global_facts": 0.31313131313131315, "machine_learning": 0.34234234234234234, "security_studies": 0.5860655737704918, "public_relations": 0.5688073394495413, "professional_psychology": 0.46153846153846156, "prehistory": 0.47678018575851394, "anatomy": 0.44029850746268656, "human_sexuality": 0.5538461538461539, "college_medicine": 0.4883720930232558, "high_school_government_and_politics": 0.6770833333333334, "college_chemistry": 0.41414141414141414, "logical_fallacies": 0.5555555555555556, "high_school_geography": 0.6700507614213198, "elementary_mathematics": 0.32625994694960214, "human_aging": 0.5180180180180181, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.6893382352941176, "formal_logic": 0.352, "high_school_statistics": 0.4232558139534884, "international_law": 0.6666666666666666, "high_school_mathematics": 0.31226765799256506, "high_school_computer_science": 0.47474747474747475, "conceptual_physics": 0.42735042735042733, "miscellaneous": 0.6508951406649617, "high_school_chemistry": 0.3217821782178218, "marketing": 0.759656652360515, "professional_law": 0.37638617090671883, "management": 0.696078431372549, "college_physics": 0.32673267326732675, "jurisprudence": 0.5887850467289719, "world_religions": 0.5764705882352941, "sociology": 0.74, "us_foreign_policy": 0.6767676767676768, "high_school_macroeconomics": 0.5347043701799485, "computer_security": 0.6161616161616161, "moral_scenarios": 0.22706935123042504, "moral_disputes": 0.5768115942028985, "electrical_engineering": 0.4166666666666667, "astronomy": 0.5165562913907285, "college_biology": 0.4755244755244755 } } }, "c_eval": { "prompt_1": { "accuracy": 0.25705794947994054 }, "prompt_2": { "accuracy": 0.2473997028231798 }, "prompt_3": { "accuracy": 0.25185735512630014 }, "prompt_4": { "accuracy": 0.2473997028231798 }, "prompt_5": { "accuracy": 0.24665676077265974 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.26214196762141967, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4583333333333333, "computer_architecture": 0.23076923076923078, "college_programming": 0.3333333333333333, "college_physics": 0.125, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.19047619047619047, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.10714285714285714, "college_economics": 0.21666666666666667, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.22448979591836735, "high_school_politics": 0.08333333333333333, "high_school_geography": 0.25, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.2962962962962963, "civil_servant": 0.25, "sports_science": 0.2916666666666667, "plant_protection": 0.18518518518518517, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3148148148148148, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.3148148148148148, "physician": 0.25925925925925924 } }, "prompt_2": { "accuracy": 0.27210460772104605, "category_acc": { "computer_network": 0.16666666666666666, "operating_system": 0.4166666666666667, "computer_architecture": 0.2692307692307692, "college_programming": 0.3333333333333333, "college_physics": 0.16666666666666666, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.21428571428571427, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.26666666666666666, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.24489795918367346, "high_school_politics": 0.08333333333333333, "high_school_geography": 0.25, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.2962962962962963, "civil_servant": 0.2692307692307692, "sports_science": 0.2916666666666667, "plant_protection": 0.18518518518518517, "basic_medicine": 0.375, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.3148148148148148, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.3148148148148148, "physician": 0.2962962962962963 } }, "prompt_3": { "accuracy": 0.26899128268991285, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.5, "computer_architecture": 0.19230769230769232, "college_programming": 0.30952380952380953, "college_physics": 0.16666666666666666, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.21428571428571427, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.21428571428571427, "college_economics": 0.25, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.24489795918367346, "high_school_politics": 0.08333333333333333, "high_school_geography": 0.25, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.2962962962962963, "civil_servant": 0.2692307692307692, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.2962962962962963, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3148148148148148, "physician": 0.2777777777777778 } }, "prompt_4": { "accuracy": 0.2683686176836862, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.375, "computer_architecture": 0.19230769230769232, "college_programming": 0.30952380952380953, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.25, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.14285714285714285, "college_economics": 0.21666666666666667, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.2647058823529412, "teacher_qualification": 0.24489795918367346, "high_school_politics": 0.08333333333333333, "high_school_geography": 0.25, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.35714285714285715, "high_school_chinese": 0.25, "high_school_history": 0.12, "middle_school_history": 0.2962962962962963, "civil_servant": 0.25, "sports_science": 0.375, "plant_protection": 0.2222222222222222, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3137254901960784, "accountant": 0.2962962962962963, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.3055555555555556, "tax_accountant": 0.3148148148148148, "physician": 0.2962962962962963 } }, "prompt_5": { "accuracy": 0.26650062266500624, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.4583333333333333, "computer_architecture": 0.3076923076923077, "college_programming": 0.2619047619047619, "college_physics": 0.16666666666666666, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.16666666666666666, "metrology_engineer": 0.2413793103448276, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.25, "high_school_chemistry": 0.375, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.21666666666666667, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.24489795918367346, "high_school_politics": 0.08333333333333333, "high_school_geography": 0.25, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.2962962962962963, "civil_servant": 0.25, "sports_science": 0.2916666666666667, "plant_protection": 0.18518518518518517, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.3148148148148148, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3148148148148148, "physician": 0.2777777777777778 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.2616487455197133 }, "prompt_2": { "accuracy": 0.23297491039426524 }, "prompt_3": { "accuracy": 0.25806451612903225 }, "prompt_4": { "accuracy": 0.2724014336917563 }, "prompt_5": { "accuracy": 0.27956989247311825 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.24563978587463306, "category_acc": { "agronomy": 0.2485207100591716, "anatomy": 0.21621621621621623, "ancient_chinese": 0.24390243902439024, "arts": 0.225, "astronomy": 0.3090909090909091, "business_ethics": 0.2583732057416268, "chinese_civil_service_exam": 0.26875, "chinese_driving_rule": 0.25190839694656486, "chinese_food_culture": 0.27205882352941174, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.23219814241486067, "chinese_literature": 0.20098039215686275, "chinese_teacher_qualification": 0.21787709497206703, "clinical_knowledge": 0.24050632911392406, "college_actuarial_science": 0.2169811320754717, "college_education": 0.21495327102803738, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.23148148148148148, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.27358490566037735, "college_medicine": 0.25274725274725274, "computer_science": 0.22058823529411764, "computer_security": 0.2631578947368421, "conceptual_physics": 0.20408163265306123, "construction_project_management": 0.30935251798561153, "economics": 0.18238993710691823, "education": 0.25766871165644173, "electrical_engineering": 0.22674418604651161, "elementary_chinese": 0.1865079365079365, "elementary_commonsense": 0.2676767676767677, "elementary_information_and_technology": 0.28991596638655465, "elementary_mathematics": 0.23043478260869565, "ethnology": 0.3037037037037037, "food_science": 0.26573426573426573, "genetics": 0.3125, "global_facts": 0.2348993288590604, "high_school_biology": 0.20710059171597633, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.2288135593220339, "high_school_mathematics": 0.21341463414634146, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.1958041958041958, "human_sexuality": 0.1984126984126984, "international_law": 0.2594594594594595, "journalism": 0.20348837209302326, "jurisprudence": 0.25060827250608275, "legal_and_moral_basis": 0.27102803738317754, "logical": 0.24390243902439024, "machine_learning": 0.2540983606557377, "management": 0.22380952380952382, "marketing": 0.2111111111111111, "marxist_theory": 0.21693121693121692, "modern_chinese": 0.31896551724137934, "nutrition": 0.23448275862068965, "philosophy": 0.2571428571428571, "professional_accounting": 0.2857142857142857, "professional_law": 0.22748815165876776, "professional_medicine": 0.25, "professional_psychology": 0.24568965517241378, "public_relations": 0.28735632183908044, "security_study": 0.2222222222222222, "sociology": 0.19911504424778761, "sports_science": 0.3393939393939394, "traditional_chinese_medicine": 0.2648648648648649, "virology": 0.22485207100591717, "world_history": 0.2732919254658385, "world_religions": 0.275 } }, "prompt_2": { "accuracy": 0.2487480573303402, "category_acc": { "agronomy": 0.2958579881656805, "anatomy": 0.25, "ancient_chinese": 0.21341463414634146, "arts": 0.225, "astronomy": 0.296969696969697, "business_ethics": 0.27751196172248804, "chinese_civil_service_exam": 0.25625, "chinese_driving_rule": 0.24427480916030533, "chinese_food_culture": 0.3014705882352941, "chinese_foreign_policy": 0.2616822429906542, "chinese_history": 0.24148606811145512, "chinese_literature": 0.2696078431372549, "chinese_teacher_qualification": 0.29608938547486036, "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.2169811320754717, "college_education": 0.24299065420560748, "college_engineering_hydrology": 0.3018867924528302, "college_law": 0.16666666666666666, "college_mathematics": 0.3333333333333333, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.20512820512820512, "computer_science": 0.27941176470588236, "computer_security": 0.25146198830409355, "conceptual_physics": 0.20408163265306123, "construction_project_management": 0.26618705035971224, "economics": 0.23270440251572327, "education": 0.24539877300613497, "electrical_engineering": 0.3023255813953488, "elementary_chinese": 0.21825396825396826, "elementary_commonsense": 0.23232323232323232, "elementary_information_and_technology": 0.24369747899159663, "elementary_mathematics": 0.26521739130434785, "ethnology": 0.22962962962962963, "food_science": 0.25874125874125875, "genetics": 0.2727272727272727, "global_facts": 0.2348993288590604, "high_school_biology": 0.20710059171597633, "high_school_chemistry": 0.24242424242424243, "high_school_geography": 0.211864406779661, "high_school_mathematics": 0.20121951219512196, "high_school_physics": 0.2, "high_school_politics": 0.24475524475524477, "human_sexuality": 0.2857142857142857, "international_law": 0.2918918918918919, "journalism": 0.23255813953488372, "jurisprudence": 0.26520681265206814, "legal_and_moral_basis": 0.205607476635514, "logical": 0.22764227642276422, "machine_learning": 0.22131147540983606, "management": 0.19523809523809524, "marketing": 0.2388888888888889, "marxist_theory": 0.1693121693121693, "modern_chinese": 0.23275862068965517, "nutrition": 0.2689655172413793, "philosophy": 0.26666666666666666, "professional_accounting": 0.21142857142857144, "professional_law": 0.2559241706161137, "professional_medicine": 0.2579787234042553, "professional_psychology": 0.2672413793103448, "public_relations": 0.3160919540229885, "security_study": 0.22962962962962963, "sociology": 0.22566371681415928, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.25443786982248523, "world_history": 0.2484472049689441, "world_religions": 0.33125 } }, "prompt_3": { "accuracy": 0.25755482645484373, "category_acc": { "agronomy": 0.27218934911242604, "anatomy": 0.2702702702702703, "ancient_chinese": 0.2073170731707317, "arts": 0.26875, "astronomy": 0.2606060606060606, "business_ethics": 0.2822966507177033, "chinese_civil_service_exam": 0.2625, "chinese_driving_rule": 0.29770992366412213, "chinese_food_culture": 0.29411764705882354, "chinese_foreign_policy": 0.205607476635514, "chinese_history": 0.25077399380804954, "chinese_literature": 0.2696078431372549, "chinese_teacher_qualification": 0.20670391061452514, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.25471698113207547, "college_education": 0.27102803738317754, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.28703703703703703, "college_mathematics": 0.3333333333333333, "college_medical_statistics": 0.3018867924528302, "college_medicine": 0.2490842490842491, "computer_science": 0.27450980392156865, "computer_security": 0.30994152046783624, "conceptual_physics": 0.272108843537415, "construction_project_management": 0.26618705035971224, "economics": 0.2389937106918239, "education": 0.2392638036809816, "electrical_engineering": 0.27906976744186046, "elementary_chinese": 0.26587301587301587, "elementary_commonsense": 0.2474747474747475, "elementary_information_and_technology": 0.29411764705882354, "elementary_mathematics": 0.29130434782608694, "ethnology": 0.22962962962962963, "food_science": 0.27972027972027974, "genetics": 0.23295454545454544, "global_facts": 0.22818791946308725, "high_school_biology": 0.23076923076923078, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.22033898305084745, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.23776223776223776, "human_sexuality": 0.20634920634920634, "international_law": 0.1891891891891892, "journalism": 0.20930232558139536, "jurisprudence": 0.25304136253041365, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.25203252032520324, "machine_learning": 0.23770491803278687, "management": 0.2714285714285714, "marketing": 0.2833333333333333, "marxist_theory": 0.24338624338624337, "modern_chinese": 0.2672413793103448, "nutrition": 0.2413793103448276, "philosophy": 0.22857142857142856, "professional_accounting": 0.28, "professional_law": 0.21800947867298578, "professional_medicine": 0.2526595744680851, "professional_psychology": 0.22413793103448276, "public_relations": 0.25862068965517243, "security_study": 0.3037037037037037, "sociology": 0.22123893805309736, "sports_science": 0.2606060606060606, "traditional_chinese_medicine": 0.33513513513513515, "virology": 0.2781065088757396, "world_history": 0.2857142857142857, "world_religions": 0.2625 } }, "prompt_4": { "accuracy": 0.2485753755828009, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.23648648648648649, "ancient_chinese": 0.21951219512195122, "arts": 0.25, "astronomy": 0.24242424242424243, "business_ethics": 0.3062200956937799, "chinese_civil_service_exam": 0.3125, "chinese_driving_rule": 0.21374045801526717, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.29906542056074764, "chinese_history": 0.2260061919504644, "chinese_literature": 0.23039215686274508, "chinese_teacher_qualification": 0.22905027932960895, "clinical_knowledge": 0.270042194092827, "college_actuarial_science": 0.20754716981132076, "college_education": 0.17757009345794392, "college_engineering_hydrology": 0.2358490566037736, "college_law": 0.23148148148148148, "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.2641509433962264, "college_medicine": 0.21611721611721613, "computer_science": 0.25, "computer_security": 0.29239766081871343, "conceptual_physics": 0.30612244897959184, "construction_project_management": 0.26618705035971224, "economics": 0.25157232704402516, "education": 0.2883435582822086, "electrical_engineering": 0.23837209302325582, "elementary_chinese": 0.27380952380952384, "elementary_commonsense": 0.22727272727272727, "elementary_information_and_technology": 0.18067226890756302, "elementary_mathematics": 0.24782608695652175, "ethnology": 0.26666666666666666, "food_science": 0.27972027972027974, "genetics": 0.29545454545454547, "global_facts": 0.2080536912751678, "high_school_biology": 0.28402366863905326, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.22033898305084745, "high_school_mathematics": 0.2073170731707317, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.25874125874125875, "human_sexuality": 0.2698412698412698, "international_law": 0.2972972972972973, "journalism": 0.20930232558139536, "jurisprudence": 0.25547445255474455, "legal_and_moral_basis": 0.22429906542056074, "logical": 0.21138211382113822, "machine_learning": 0.26229508196721313, "management": 0.24285714285714285, "marketing": 0.2777777777777778, "marxist_theory": 0.24867724867724866, "modern_chinese": 0.21551724137931033, "nutrition": 0.2482758620689655, "philosophy": 0.2857142857142857, "professional_accounting": 0.16, "professional_law": 0.22274881516587677, "professional_medicine": 0.23138297872340424, "professional_psychology": 0.28879310344827586, "public_relations": 0.3218390804597701, "security_study": 0.2074074074074074, "sociology": 0.22123893805309736, "sports_science": 0.2, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.23668639053254437, "world_history": 0.2422360248447205, "world_religions": 0.31875 } }, "prompt_5": { "accuracy": 0.25392850975651876, "category_acc": { "agronomy": 0.28994082840236685, "anatomy": 0.2702702702702703, "ancient_chinese": 0.2621951219512195, "arts": 0.25, "astronomy": 0.2727272727272727, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.28125, "chinese_driving_rule": 0.2748091603053435, "chinese_food_culture": 0.27941176470588236, "chinese_foreign_policy": 0.205607476635514, "chinese_history": 0.24148606811145512, "chinese_literature": 0.28431372549019607, "chinese_teacher_qualification": 0.2122905027932961, "clinical_knowledge": 0.22784810126582278, "college_actuarial_science": 0.19811320754716982, "college_education": 0.18691588785046728, "college_engineering_hydrology": 0.27358490566037735, "college_law": 0.2777777777777778, "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.19811320754716982, "college_medicine": 0.2564102564102564, "computer_science": 0.21568627450980393, "computer_security": 0.23391812865497075, "conceptual_physics": 0.2925170068027211, "construction_project_management": 0.23741007194244604, "economics": 0.22641509433962265, "education": 0.26993865030674846, "electrical_engineering": 0.2558139534883721, "elementary_chinese": 0.2222222222222222, "elementary_commonsense": 0.2828282828282828, "elementary_information_and_technology": 0.25210084033613445, "elementary_mathematics": 0.25217391304347825, "ethnology": 0.28888888888888886, "food_science": 0.2727272727272727, "genetics": 0.3068181818181818, "global_facts": 0.2214765100671141, "high_school_biology": 0.23668639053254437, "high_school_chemistry": 0.20454545454545456, "high_school_geography": 0.2796610169491525, "high_school_mathematics": 0.23170731707317074, "high_school_physics": 0.22727272727272727, "high_school_politics": 0.21678321678321677, "human_sexuality": 0.25396825396825395, "international_law": 0.21621621621621623, "journalism": 0.22093023255813954, "jurisprudence": 0.25547445255474455, "legal_and_moral_basis": 0.24766355140186916, "logical": 0.2845528455284553, "machine_learning": 0.27049180327868855, "management": 0.26666666666666666, "marketing": 0.22777777777777777, "marxist_theory": 0.20105820105820105, "modern_chinese": 0.22413793103448276, "nutrition": 0.22758620689655173, "philosophy": 0.23809523809523808, "professional_accounting": 0.2857142857142857, "professional_law": 0.26540284360189575, "professional_medicine": 0.26595744680851063, "professional_psychology": 0.27586206896551724, "public_relations": 0.27011494252873564, "security_study": 0.2814814814814815, "sociology": 0.30973451327433627, "sports_science": 0.23636363636363636, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.3076923076923077, "world_history": 0.2670807453416149, "world_religions": 0.2875 } } }, "zbench": { "prompt_1": { "accuracy": 0.5454545454545454 }, "prompt_2": { "accuracy": 0.3939393939393939 }, "prompt_3": { "accuracy": 0.42424242424242425 }, "prompt_4": { "accuracy": 0.36363636363636365 }, "prompt_5": { "accuracy": 0.42424242424242425 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.36818181818181817 }, "prompt_2": { "accuracy": 0.425 }, "prompt_3": { "accuracy": 0.4090909090909091 }, "prompt_4": { "accuracy": 0.4431818181818182 }, "prompt_5": { "accuracy": 0.4590909090909091 } }, "ocnli": { "prompt_1": { "accuracy": 0.3257627118644068 }, "prompt_2": { "accuracy": 0.31966101694915255 }, "prompt_3": { "accuracy": 0.31559322033898307 }, "prompt_4": { "accuracy": 0.32610169491525426 }, "prompt_5": { "accuracy": 0.3247457627118644 } }, "c3": { "prompt_1": { "accuracy": 0.275243081525804 }, "prompt_2": { "accuracy": 0.27599102468212416 }, "prompt_3": { "accuracy": 0.27898279730740466 }, "prompt_4": { "accuracy": 0.27225130890052357 }, "prompt_5": { "accuracy": 0.2801047120418848 } }, "dream": { "prompt_1": { "accuracy": 0.9216070553650172 }, "prompt_2": { "accuracy": 0.924546790788829 }, "prompt_3": { "accuracy": 0.9220970112689858 }, "prompt_4": { "accuracy": 0.9230769230769231 }, "prompt_5": { "accuracy": 0.9240568348848603 } }, "samsum": { "prompt_1": { "rouge1": 0.5283994525332824, "rouge2": 0.28407900462604924, "rougeL": 0.4438398405240641, "avg_rouge": 0.41877276589446516 }, "prompt_2": { "rouge1": 0.5310040504588704, "rouge2": 0.2889554535263174, "rougeL": 0.4448795824256869, "avg_rouge": 0.4216130288036249 }, "prompt_3": { "rouge1": 0.5301678345319392, "rouge2": 0.2888796755768626, "rougeL": 0.44474721577330006, "avg_rouge": 0.42126490862736726 }, "prompt_4": { "rouge1": 0.5289238690020692, "rouge2": 0.286647554142563, "rougeL": 0.4426701857145179, "avg_rouge": 0.4194138696197167 }, "prompt_5": { "rouge1": 0.5286747119788637, "rouge2": 0.28676322097611173, "rougeL": 0.44334451795455654, "avg_rouge": 0.4195941503031773 } }, "dialogsum": { "prompt_1": { "rouge1": 0.2348990714241612, "rouge2": 0.0654692982269716, "rougeL": 0.17545716448016013, "avg_rouge": 0.15860851137709764 }, "prompt_2": { "rouge1": 0.23235043239609107, "rouge2": 0.06516831879832731, "rougeL": 0.17253374137518865, "avg_rouge": 0.156684164189869 }, "prompt_3": { "rouge1": 0.2318659794631441, "rouge2": 0.064963463287211, "rougeL": 0.1716820969020561, "avg_rouge": 0.1561705132174704 }, "prompt_4": { "rouge1": 0.2347594030365569, "rouge2": 0.06517399918190821, "rougeL": 0.17496958526054637, "avg_rouge": 0.15830099582633717 }, "prompt_5": { "rouge1": 0.23476352815147197, "rouge2": 0.06338707364815505, "rougeL": 0.18093639777893736, "avg_rouge": 0.15969566652618813 } }, "sst2": { "prompt_1": { "accuracy": 0.9598623853211009 }, "prompt_2": { "accuracy": 0.9598623853211009 }, "prompt_3": { "accuracy": 0.9541284403669725 }, "prompt_4": { "accuracy": 0.9587155963302753 }, "prompt_5": { "accuracy": 0.9541284403669725 } }, "cola": { "prompt_1": { "accuracy": 0.8072866730584851 }, "prompt_2": { "accuracy": 0.8072866730584851 }, "prompt_3": { "accuracy": 0.8015340364333653 }, "prompt_4": { "accuracy": 0.8015340364333653 }, "prompt_5": { "accuracy": 0.8034515819750719 } }, "qqp": { "prompt_1": { "accuracy": 0.844 }, "prompt_2": { "accuracy": 0.8665 }, "prompt_3": { "accuracy": 0.8565 }, "prompt_4": { "accuracy": 0.863 }, "prompt_5": { "accuracy": 0.864 } }, "mnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "qnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "wnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "mrpc": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.8 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "flan_t5_xxl": { "model_size": "11B", "model_link": "https://huggingface.co/google/flan-t5-xxl", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.7771008403361345, "language_acc": { "Spanish": 0.9613445378151261, "English": 0.9697478991596639, "Chinese": 0.3445378151260504, "Vietnamese": 0.8327731092436975 }, "consistency_score_2": 0.6126050420168068, "consistency_score_3": 0.44054621848739495, "consistency_score_4": 0.3025210084033613, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.973109243697479, "Spanish,Chinese": 0.33949579831932775, "Spanish,Vietnamese": 0.8243697478991596, "English,Chinese": 0.33865546218487397, "English,Vietnamese": 0.8294117647058824, "Chinese,Vietnamese": 0.37058823529411766 }, "3_combine": { "Spanish,English,Chinese": 0.3319327731092437, "Spanish,English,Vietnamese": 0.8184873949579832, "Spanish,Chinese,Vietnamese": 0.3050420168067227, "English,Chinese,Vietnamese": 0.3067226890756303 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3025210084033613 } }, "AC3_2": 0.6851174755123539, "AC3_3": 0.5623120987063642, "AC3_4": 0.43550309787133057 }, "prompt_2": { "overall_acc": 0.7785714285714286, "language_acc": { "Spanish": 0.9621848739495799, "English": 0.9689075630252101, "Chinese": 0.34705882352941175, "Vietnamese": 0.8361344537815126 }, "consistency_score_2": 0.6145658263305322, "consistency_score_3": 0.4430672268907563, "consistency_score_4": 0.3042016806722689, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9747899159663865, "Spanish,Chinese": 0.3420168067226891, "Spanish,Vietnamese": 0.8319327731092437, "English,Chinese": 0.3420168067226891, "English,Vietnamese": 0.8302521008403362, "Chinese,Vietnamese": 0.3663865546218487 }, "3_combine": { "Spanish,English,Chinese": 0.3352941176470588, "Spanish,English,Vietnamese": 0.8226890756302521, "Spanish,Chinese,Vietnamese": 0.30756302521008405, "English,Chinese,Vietnamese": 0.3067226890756303 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3042016806722689 } }, "AC3_2": 0.6869149348197849, "AC3_3": 0.5647488023122037, "AC3_4": 0.43747436106608834 }, "prompt_3": { "overall_acc": 0.7779411764705882, "language_acc": { "Spanish": 0.9605042016806723, "English": 0.9689075630252101, "Chinese": 0.3487394957983193, "Vietnamese": 0.8336134453781513 }, "consistency_score_2": 0.6162464985994397, "consistency_score_3": 0.4445378151260504, "consistency_score_4": 0.30840336134453783, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9739495798319328, "Spanish,Chinese": 0.34369747899159664, "Spanish,Vietnamese": 0.8260504201680672, "English,Chinese": 0.3445378151260504, "English,Vietnamese": 0.8277310924369747, "Chinese,Vietnamese": 0.3815126050420168 }, "3_combine": { "Spanish,English,Chinese": 0.3369747899159664, "Spanish,English,Vietnamese": 0.8184873949579832, "Spanish,Chinese,Vietnamese": 0.31176470588235294, "English,Chinese,Vietnamese": 0.31092436974789917 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.30840336134453783 } }, "AC3_2": 0.6877173491838008, "AC3_3": 0.5657754010232376, "AC3_4": 0.441700888398259 }, "prompt_4": { "overall_acc": 0.7779411764705882, "language_acc": { "Spanish": 0.9613445378151261, "English": 0.9705882352941176, "Chinese": 0.3487394957983193, "Vietnamese": 0.83109243697479 }, "consistency_score_2": 0.6147058823529411, "consistency_score_3": 0.4432773109243697, "consistency_score_4": 0.3058823529411765, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9747899159663865, "Spanish,Chinese": 0.3453781512605042, "Spanish,Vietnamese": 0.8243697478991596, "English,Chinese": 0.34369747899159664, "English,Vietnamese": 0.826890756302521, "Chinese,Vietnamese": 0.373109243697479 }, "3_combine": { "Spanish,English,Chinese": 0.3378151260504202, "Spanish,English,Vietnamese": 0.8168067226890756, "Spanish,Chinese,Vietnamese": 0.3092436974789916, "English,Chinese,Vietnamese": 0.3092436974789916 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3058823529411765 } }, "AC3_2": 0.6867569413756202, "AC3_3": 0.564753443047834, "AC3_4": 0.4391092664612012 }, "prompt_5": { "overall_acc": 0.7655462184873949, "language_acc": { "Spanish": 0.9487394957983193, "English": 0.9554621848739496, "Chinese": 0.346218487394958, "Vietnamese": 0.8117647058823529 }, "consistency_score_2": 0.5976190476190476, "consistency_score_3": 0.4262605042016807, "consistency_score_4": 0.28823529411764703, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9512605042016806, "Spanish,Chinese": 0.3420168067226891, "Spanish,Vietnamese": 0.8042016806722689, "English,Chinese": 0.34285714285714286, "English,Vietnamese": 0.8033613445378152, "Chinese,Vietnamese": 0.3420168067226891 }, "3_combine": { "Spanish,English,Chinese": 0.3302521008403361, "Spanish,English,Vietnamese": 0.7865546218487395, "Spanish,Chinese,Vietnamese": 0.29327731092436976, "English,Chinese,Vietnamese": 0.2949579831932773 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.28823529411764703 } }, "AC3_2": 0.6712392302570803, "AC3_3": 0.5476091229263507, "AC3_4": 0.41879163144538817 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.3771428571428571, "language_acc": { "Filipino": 0.3466666666666667, "Vietnamese": 0.2866666666666667, "Chinese": 0.22, "Spanish": 0.44, "Malay": 0.3333333333333333, "Indonesian": 0.42, "English": 0.5933333333333334 }, "consistency_score_2": 0.41587301587301584, "consistency_score_3": 0.21504761904761907, "consistency_score_4": 0.12685714285714284, "consistency_score_5": 0.0803174603174603, "consistency_score_6": 0.05238095238095238, "consistency_score_7": 0.03333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.3933333333333333, "Filipino,Chinese": 0.30666666666666664, "Filipino,Spanish": 0.41333333333333333, "Filipino,Malay": 0.4866666666666667, "Filipino,Indonesian": 0.49333333333333335, "Filipino,English": 0.4266666666666667, "Vietnamese,Chinese": 0.5066666666666667, "Vietnamese,Spanish": 0.34, "Vietnamese,Malay": 0.32, "Vietnamese,Indonesian": 0.36, "Vietnamese,English": 0.32666666666666666, "Chinese,Spanish": 0.26, "Chinese,Malay": 0.28, "Chinese,Indonesian": 0.32666666666666666, "Chinese,English": 0.2733333333333333, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.58, "Spanish,English": 0.5933333333333334, "Malay,Indonesian": 0.6133333333333333, "Malay,English": 0.4266666666666667, "Indonesian,English": 0.52 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.16666666666666666, "Filipino,Vietnamese,Spanish": 0.19333333333333333, "Filipino,Vietnamese,Malay": 0.2, "Filipino,Vietnamese,Indonesian": 0.22, "Filipino,Vietnamese,English": 0.18666666666666668, "Filipino,Chinese,Spanish": 0.13333333333333333, "Filipino,Chinese,Malay": 0.13333333333333333, "Filipino,Chinese,Indonesian": 0.15333333333333332, "Filipino,Chinese,English": 0.14666666666666667, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.2866666666666667, "Filipino,Malay,Indonesian": 0.35333333333333333, "Filipino,Malay,English": 0.26, "Filipino,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish": 0.14666666666666667, "Vietnamese,Chinese,Malay": 0.14, "Vietnamese,Chinese,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,English": 0.13333333333333333, "Vietnamese,Spanish,Malay": 0.18666666666666668, "Vietnamese,Spanish,Indonesian": 0.22666666666666666, "Vietnamese,Spanish,English": 0.21333333333333335, "Vietnamese,Malay,Indonesian": 0.22, "Vietnamese,Malay,English": 0.15333333333333332, "Vietnamese,Indonesian,English": 0.2, "Chinese,Spanish,Malay": 0.15333333333333332, "Chinese,Spanish,Indonesian": 0.18666666666666668, "Chinese,Spanish,English": 0.16, "Chinese,Malay,Indonesian": 0.19333333333333333, "Chinese,Malay,English": 0.12, "Chinese,Indonesian,English": 0.16666666666666666, "Spanish,Malay,Indonesian": 0.3933333333333333, "Spanish,Malay,English": 0.32, "Spanish,Indonesian,English": 0.38666666666666666, "Malay,Indonesian,English": 0.34 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.08, "Filipino,Vietnamese,Chinese,Indonesian": 0.08666666666666667, "Filipino,Vietnamese,Chinese,English": 0.06666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay": 0.1, "Filipino,Chinese,Spanish,Indonesian": 0.1, "Filipino,Chinese,Spanish,English": 0.08666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.11333333333333333, "Filipino,Chinese,Malay,English": 0.08, "Filipino,Chinese,Indonesian,English": 0.08666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.22, "Filipino,Spanish,Indonesian,English": 0.23333333333333334, "Filipino,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay": 0.08, "Vietnamese,Chinese,Spanish,Indonesian": 0.10666666666666667, "Vietnamese,Chinese,Spanish,English": 0.08, "Vietnamese,Chinese,Malay,Indonesian": 0.10666666666666667, "Vietnamese,Chinese,Malay,English": 0.06, "Vietnamese,Chinese,Indonesian,English": 0.08, "Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, "Vietnamese,Spanish,Malay,English": 0.13333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Malay,Indonesian,English": 0.12666666666666668, "Chinese,Spanish,Malay,Indonesian": 0.14, "Chinese,Spanish,Malay,English": 0.09333333333333334, "Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Chinese,Malay,Indonesian,English": 0.10666666666666667, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.06, "Filipino,Vietnamese,Chinese,Spanish,English": 0.04, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.04, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.04, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12, "Filipino,Vietnamese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.06666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.07333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.04666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.05333333333333334, "Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.03333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.04, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.03333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.03333333333333333 } }, "AC3_2": 0.395562163967047, "AC3_3": 0.2739107659327013, "AC3_4": 0.18985422736757826, "AC3_5": 0.13243184293724472, "AC3_6": 0.09198606269635422, "AC3_7": 0.06125290021709616 }, "prompt_2": { "overall_acc": 0.3857142857142857, "language_acc": { "Filipino": 0.38666666666666666, "Vietnamese": 0.2733333333333333, "Chinese": 0.22666666666666666, "Spanish": 0.4533333333333333, "Malay": 0.3466666666666667, "Indonesian": 0.42, "English": 0.5933333333333334 }, "consistency_score_2": 0.43015873015873013, "consistency_score_3": 0.2327619047619048, "consistency_score_4": 0.14266666666666664, "consistency_score_5": 0.09301587301587302, "consistency_score_6": 0.06190476190476191, "consistency_score_7": 0.04, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4, "Filipino,Chinese": 0.3333333333333333, "Filipino,Spanish": 0.4266666666666667, "Filipino,Malay": 0.49333333333333335, "Filipino,Indonesian": 0.48, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.4, "Vietnamese,Spanish": 0.3466666666666667, "Vietnamese,Malay": 0.36, "Vietnamese,Indonesian": 0.37333333333333335, "Vietnamese,English": 0.3333333333333333, "Chinese,Spanish": 0.35333333333333333, "Chinese,Malay": 0.35333333333333333, "Chinese,Indonesian": 0.38666666666666666, "Chinese,English": 0.31333333333333335, "Spanish,Malay": 0.47333333333333333, "Spanish,Indonesian": 0.58, "Spanish,English": 0.58, "Malay,Indonesian": 0.6133333333333333, "Malay,English": 0.43333333333333335, "Indonesian,English": 0.5333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.16, "Filipino,Vietnamese,Spanish": 0.2, "Filipino,Vietnamese,Malay": 0.22, "Filipino,Vietnamese,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,English": 0.20666666666666667, "Filipino,Chinese,Spanish": 0.18, "Filipino,Chinese,Malay": 0.16666666666666666, "Filipino,Chinese,Indonesian": 0.18, "Filipino,Chinese,English": 0.17333333333333334, "Filipino,Spanish,Malay": 0.2866666666666667, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.3, "Filipino,Malay,Indonesian": 0.34, "Filipino,Malay,English": 0.2866666666666667, "Filipino,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish": 0.16, "Vietnamese,Chinese,Malay": 0.15333333333333332, "Vietnamese,Chinese,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,English": 0.13333333333333333, "Vietnamese,Spanish,Malay": 0.19333333333333333, "Vietnamese,Spanish,Indonesian": 0.25333333333333335, "Vietnamese,Spanish,English": 0.22, "Vietnamese,Malay,Indonesian": 0.24, "Vietnamese,Malay,English": 0.16666666666666666, "Vietnamese,Indonesian,English": 0.22, "Chinese,Spanish,Malay": 0.20666666666666667, "Chinese,Spanish,Indonesian": 0.24666666666666667, "Chinese,Spanish,English": 0.20666666666666667, "Chinese,Malay,Indonesian": 0.24666666666666667, "Chinese,Malay,English": 0.16, "Chinese,Indonesian,English": 0.2, "Spanish,Malay,Indonesian": 0.3933333333333333, "Spanish,Malay,English": 0.31333333333333335, "Spanish,Indonesian,English": 0.3933333333333333, "Malay,Indonesian,English": 0.3466666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Indonesian": 0.08666666666666667, "Filipino,Vietnamese,Chinese,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.14, "Filipino,Vietnamese,Spanish,Indonesian": 0.16, "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay": 0.12, "Filipino,Chinese,Spanish,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,English": 0.12, "Filipino,Chinese,Malay,Indonesian": 0.12, "Filipino,Chinese,Malay,English": 0.10666666666666667, "Filipino,Chinese,Indonesian,English": 0.11333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Spanish,Malay,English": 0.22, "Filipino,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.1, "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Vietnamese,Chinese,Spanish,English": 0.08666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, "Vietnamese,Chinese,Malay,English": 0.06666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Spanish,Malay,English": 0.14666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Vietnamese,Malay,Indonesian,English": 0.14666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.18, "Chinese,Spanish,Malay,English": 0.14, "Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Chinese,Malay,Indonesian,English": 0.14666666666666667, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.04666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.04666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.1, "Filipino,Chinese,Spanish,Malay,English": 0.09333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.06666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.06, "Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.04666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.04666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.04, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.1, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04 } }, "AC3_2": 0.40672595881618495, "AC3_3": 0.290325135242776, "AC3_4": 0.20829127609612366, "AC3_5": 0.1498863205446666, "AC3_6": 0.1066869300673511, "AC3_7": 0.07248322145948381 }, "prompt_3": { "overall_acc": 0.3876190476190477, "language_acc": { "Filipino": 0.38666666666666666, "Vietnamese": 0.2866666666666667, "Chinese": 0.25333333333333335, "Spanish": 0.4533333333333333, "Malay": 0.3466666666666667, "Indonesian": 0.4, "English": 0.5866666666666667 }, "consistency_score_2": 0.4428571428571428, "consistency_score_3": 0.2472380952380952, "consistency_score_4": 0.15333333333333332, "consistency_score_5": 0.09936507936507938, "consistency_score_6": 0.06476190476190476, "consistency_score_7": 0.04, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.44666666666666666, "Filipino,Chinese": 0.3933333333333333, "Filipino,Spanish": 0.47333333333333333, "Filipino,Malay": 0.4666666666666667, "Filipino,Indonesian": 0.47333333333333333, "Filipino,English": 0.48, "Vietnamese,Chinese": 0.43333333333333335, "Vietnamese,Spanish": 0.35333333333333333, "Vietnamese,Malay": 0.36666666666666664, "Vietnamese,Indonesian": 0.38666666666666666, "Vietnamese,English": 0.34, "Chinese,Spanish": 0.37333333333333335, "Chinese,Malay": 0.38, "Chinese,Indonesian": 0.38, "Chinese,English": 0.3333333333333333, "Spanish,Malay": 0.46, "Spanish,Indonesian": 0.5933333333333334, "Spanish,English": 0.6066666666666667, "Malay,Indonesian": 0.5933333333333334, "Malay,English": 0.44, "Indonesian,English": 0.5266666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.21333333333333335, "Filipino,Vietnamese,Spanish": 0.22666666666666666, "Filipino,Vietnamese,Malay": 0.22666666666666666, "Filipino,Vietnamese,Indonesian": 0.24, "Filipino,Vietnamese,English": 0.22, "Filipino,Chinese,Spanish": 0.24, "Filipino,Chinese,Malay": 0.2, "Filipino,Chinese,Indonesian": 0.2, "Filipino,Chinese,English": 0.20666666666666667, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.3333333333333333, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.32666666666666666, "Filipino,Malay,English": 0.2866666666666667, "Filipino,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish": 0.18, "Vietnamese,Chinese,Malay": 0.18666666666666668, "Vietnamese,Chinese,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,English": 0.14, "Vietnamese,Spanish,Malay": 0.19333333333333333, "Vietnamese,Spanish,Indonesian": 0.26, "Vietnamese,Spanish,English": 0.22666666666666666, "Vietnamese,Malay,Indonesian": 0.24, "Vietnamese,Malay,English": 0.18, "Vietnamese,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay": 0.22, "Chinese,Spanish,Indonesian": 0.26, "Chinese,Spanish,English": 0.22, "Chinese,Malay,Indonesian": 0.25333333333333335, "Chinese,Malay,English": 0.18, "Chinese,Indonesian,English": 0.21333333333333335, "Spanish,Malay,Indonesian": 0.38666666666666666, "Spanish,Malay,English": 0.31333333333333335, "Spanish,Indonesian,English": 0.4066666666666667, "Malay,Indonesian,English": 0.3466666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.12, "Filipino,Vietnamese,Chinese,Malay": 0.12, "Filipino,Vietnamese,Chinese,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay": 0.14, "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.14, "Filipino,Chinese,Malay,English": 0.12666666666666668, "Filipino,Chinese,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Spanish,Malay,English": 0.22, "Filipino,Spanish,Indonesian,English": 0.26, "Filipino,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.12, "Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Vietnamese,Chinese,Spanish,English": 0.09333333333333334, "Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, "Vietnamese,Chinese,Malay,English": 0.08, "Vietnamese,Chinese,Indonesian,English": 0.08, "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Spanish,Malay,English": 0.13333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Malay,Indonesian,English": 0.14666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Chinese,Spanish,Malay,English": 0.14666666666666667, "Chinese,Spanish,Indonesian,English": 0.18, "Chinese,Malay,Indonesian,English": 0.16, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.08, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.06, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.06, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.05333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14, "Filipino,Vietnamese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Chinese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Vietnamese,Chinese,Spanish,Malay,English": 0.06666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Vietnamese,Chinese,Malay,Indonesian,English": 0.06, "Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.06, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.04666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.04, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04 } }, "AC3_2": 0.41340104844301284, "AC3_3": 0.30190790502866666, "AC3_4": 0.21974178399693736, "AC3_5": 0.15818091509758594, "AC3_6": 0.11098145360955247, "AC3_7": 0.07251670376923329 }, "prompt_4": { "overall_acc": 0.3904761904761905, "language_acc": { "Filipino": 0.38666666666666666, "Vietnamese": 0.2866666666666667, "Chinese": 0.22, "Spanish": 0.4666666666666667, "Malay": 0.34, "Indonesian": 0.4, "English": 0.6333333333333333 }, "consistency_score_2": 0.4238095238095238, "consistency_score_3": 0.23142857142857143, "consistency_score_4": 0.148, "consistency_score_5": 0.10285714285714284, "consistency_score_6": 0.07428571428571429, "consistency_score_7": 0.05333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4066666666666667, "Filipino,Chinese": 0.31333333333333335, "Filipino,Spanish": 0.44, "Filipino,Malay": 0.4266666666666667, "Filipino,Indonesian": 0.5133333333333333, "Filipino,English": 0.48, "Vietnamese,Chinese": 0.52, "Vietnamese,Spanish": 0.38, "Vietnamese,Malay": 0.36666666666666664, "Vietnamese,Indonesian": 0.4, "Vietnamese,English": 0.35333333333333333, "Chinese,Spanish": 0.28, "Chinese,Malay": 0.2733333333333333, "Chinese,Indonesian": 0.34, "Chinese,English": 0.2733333333333333, "Spanish,Malay": 0.4666666666666667, "Spanish,Indonesian": 0.5666666666666667, "Spanish,English": 0.6066666666666667, "Malay,Indonesian": 0.5533333333333333, "Malay,English": 0.4266666666666667, "Indonesian,English": 0.5133333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.19333333333333333, "Filipino,Vietnamese,Spanish": 0.24, "Filipino,Vietnamese,Malay": 0.2, "Filipino,Vietnamese,Indonesian": 0.26, "Filipino,Vietnamese,English": 0.22666666666666666, "Filipino,Chinese,Spanish": 0.16666666666666666, "Filipino,Chinese,Malay": 0.11333333333333333, "Filipino,Chinese,Indonesian": 0.16666666666666666, "Filipino,Chinese,English": 0.16666666666666666, "Filipino,Spanish,Malay": 0.2866666666666667, "Filipino,Spanish,Indonesian": 0.3333333333333333, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.32, "Filipino,Malay,English": 0.28, "Filipino,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish": 0.18666666666666668, "Vietnamese,Chinese,Malay": 0.16666666666666666, "Vietnamese,Chinese,Indonesian": 0.2, "Vietnamese,Chinese,English": 0.15333333333333332, "Vietnamese,Spanish,Malay": 0.21333333333333335, "Vietnamese,Spanish,Indonesian": 0.2733333333333333, "Vietnamese,Spanish,English": 0.26666666666666666, "Vietnamese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Malay,English": 0.18666666666666668, "Vietnamese,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay": 0.14666666666666667, "Chinese,Spanish,Indonesian": 0.19333333333333333, "Chinese,Spanish,English": 0.18, "Chinese,Malay,Indonesian": 0.16666666666666666, "Chinese,Malay,English": 0.12, "Chinese,Indonesian,English": 0.17333333333333334, "Spanish,Malay,Indonesian": 0.36, "Spanish,Malay,English": 0.32666666666666666, "Spanish,Indonesian,English": 0.3933333333333333, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.12, "Filipino,Vietnamese,Chinese,Malay": 0.08, "Filipino,Vietnamese,Chinese,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Spanish,Indonesian": 0.2, "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Malay,English": 0.14, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.1, "Filipino,Chinese,Spanish,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,English": 0.11333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.1, "Filipino,Chinese,Malay,English": 0.08, "Filipino,Chinese,Indonesian,English": 0.10666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.23333333333333334, "Filipino,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.14, "Vietnamese,Chinese,Spanish,English": 0.12, "Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, "Vietnamese,Chinese,Malay,English": 0.08, "Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Vietnamese,Malay,Indonesian,English": 0.15333333333333332, "Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Chinese,Spanish,Malay,English": 0.1, "Chinese,Spanish,Indonesian,English": 0.14, "Chinese,Malay,Indonesian,English": 0.11333333333333333, "Spanish,Malay,Indonesian,English": 0.26 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.14, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.07333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.09333333333333334, "Filipino,Chinese,Malay,Indonesian,English": 0.08, "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, "Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.05333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 } }, "AC3_2": 0.40646059588436395, "AC3_3": 0.29061474508562635, "AC3_4": 0.21464449942954075, "AC3_5": 0.16282404850532936, "AC3_6": 0.12482435594503927, "AC3_7": 0.09384835477141472 }, "prompt_5": { "overall_acc": 0.39333333333333326, "language_acc": { "Filipino": 0.38, "Vietnamese": 0.28, "Chinese": 0.26, "Spanish": 0.4866666666666667, "Malay": 0.3333333333333333, "Indonesian": 0.4066666666666667, "English": 0.6066666666666667 }, "consistency_score_2": 0.45142857142857135, "consistency_score_3": 0.2594285714285714, "consistency_score_4": 0.1678095238095238, "consistency_score_5": 0.11746031746031749, "consistency_score_6": 0.08666666666666667, "consistency_score_7": 0.06666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.42, "Filipino,Chinese": 0.36666666666666664, "Filipino,Spanish": 0.44666666666666666, "Filipino,Malay": 0.5, "Filipino,Indonesian": 0.54, "Filipino,English": 0.48, "Vietnamese,Chinese": 0.34, "Vietnamese,Spanish": 0.36666666666666664, "Vietnamese,Malay": 0.4066666666666667, "Vietnamese,Indonesian": 0.3933333333333333, "Vietnamese,English": 0.35333333333333333, "Chinese,Spanish": 0.38, "Chinese,Malay": 0.3933333333333333, "Chinese,Indonesian": 0.4666666666666667, "Chinese,English": 0.36666666666666664, "Spanish,Malay": 0.5066666666666667, "Spanish,Indonesian": 0.5933333333333334, "Spanish,English": 0.6533333333333333, "Malay,Indonesian": 0.5933333333333334, "Malay,English": 0.44, "Indonesian,English": 0.47333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.17333333333333334, "Filipino,Vietnamese,Spanish": 0.22, "Filipino,Vietnamese,Malay": 0.24, "Filipino,Vietnamese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,English": 0.22666666666666666, "Filipino,Chinese,Spanish": 0.20666666666666667, "Filipino,Chinese,Malay": 0.22666666666666666, "Filipino,Chinese,Indonesian": 0.24666666666666667, "Filipino,Chinese,English": 0.20666666666666667, "Filipino,Spanish,Malay": 0.32, "Filipino,Spanish,Indonesian": 0.34, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.37333333333333335, "Filipino,Malay,English": 0.30666666666666664, "Filipino,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish": 0.18, "Vietnamese,Chinese,Malay": 0.19333333333333333, "Vietnamese,Chinese,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,English": 0.16666666666666666, "Vietnamese,Spanish,Malay": 0.23333333333333334, "Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian": 0.2733333333333333, "Vietnamese,Malay,English": 0.20666666666666667, "Vietnamese,Indonesian,English": 0.22, "Chinese,Spanish,Malay": 0.23333333333333334, "Chinese,Spanish,Indonesian": 0.28, "Chinese,Spanish,English": 0.24666666666666667, "Chinese,Malay,Indonesian": 0.29333333333333333, "Chinese,Malay,English": 0.2, "Chinese,Indonesian,English": 0.22, "Spanish,Malay,Indonesian": 0.4066666666666667, "Spanish,Malay,English": 0.3466666666666667, "Spanish,Indonesian,English": 0.38666666666666666, "Malay,Indonesian,English": 0.31333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.12, "Filipino,Vietnamese,Chinese,Malay": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,English": 0.12, "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Malay,English": 0.16, "Filipino,Vietnamese,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian": 0.16, "Filipino,Chinese,Spanish,English": 0.16, "Filipino,Chinese,Malay,Indonesian": 0.16666666666666666, "Filipino,Chinese,Malay,English": 0.16, "Filipino,Chinese,Indonesian,English": 0.14666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.2733333333333333, "Filipino,Spanish,Malay,English": 0.25333333333333335, "Filipino,Spanish,Indonesian,English": 0.26, "Filipino,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, "Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Vietnamese,Chinese,Malay,English": 0.12, "Vietnamese,Chinese,Indonesian,English": 0.1, "Vietnamese,Spanish,Malay,Indonesian": 0.2, "Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.18, "Vietnamese,Malay,Indonesian,English": 0.16, "Chinese,Spanish,Malay,Indonesian": 0.2, "Chinese,Spanish,Malay,English": 0.15333333333333332, "Chinese,Spanish,Indonesian,English": 0.18666666666666668, "Chinese,Malay,Indonesian,English": 0.15333333333333332, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.08, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.14, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 } }, "AC3_2": 0.42038331449364114, "AC3_3": 0.3126466296573856, "AC3_4": 0.23525231948738146, "AC3_5": 0.18089910914212767, "AC3_6": 0.142037037007446, "AC3_7": 0.11400966181096407 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.37743506493506496, "language_acc": { "Indonesian": 0.4034090909090909, "English": 0.6022727272727273, "Filipino": 0.2897727272727273, "Spanish": 0.45454545454545453, "Chinese": 0.24431818181818182, "Malay": 0.3522727272727273, "Vietnamese": 0.29545454545454547 }, "consistency_score_2": 0.40205627705627706, "consistency_score_3": 0.20129870129870134, "consistency_score_4": 0.11347402597402598, "consistency_score_5": 0.06953463203463203, "consistency_score_6": 0.04626623376623377, "consistency_score_7": 0.03409090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4602272727272727, "Indonesian,Filipino": 0.48295454545454547, "Indonesian,Spanish": 0.5056818181818182, "Indonesian,Chinese": 0.3409090909090909, "Indonesian,Malay": 0.5397727272727273, "Indonesian,Vietnamese": 0.32954545454545453, "English,Filipino": 0.3409090909090909, "English,Spanish": 0.6647727272727273, "English,Chinese": 0.2215909090909091, "English,Malay": 0.42045454545454547, "English,Vietnamese": 0.29545454545454547, "Filipino,Spanish": 0.4034090909090909, "Filipino,Chinese": 0.2784090909090909, "Filipino,Malay": 0.48863636363636365, "Filipino,Vietnamese": 0.3693181818181818, "Spanish,Chinese": 0.25, "Spanish,Malay": 0.4659090909090909, "Spanish,Vietnamese": 0.3465909090909091, "Chinese,Malay": 0.3465909090909091, "Chinese,Vietnamese": 0.4943181818181818, "Malay,Vietnamese": 0.3977272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.23295454545454544, "Indonesian,English,Spanish": 0.36363636363636365, "Indonesian,English,Chinese": 0.13636363636363635, "Indonesian,English,Malay": 0.2840909090909091, "Indonesian,English,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish": 0.2727272727272727, "Indonesian,Filipino,Chinese": 0.17045454545454544, "Indonesian,Filipino,Malay": 0.3181818181818182, "Indonesian,Filipino,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese": 0.1534090909090909, "Indonesian,Spanish,Malay": 0.3181818181818182, "Indonesian,Spanish,Vietnamese": 0.1875, "Indonesian,Chinese,Malay": 0.21022727272727273, "Indonesian,Chinese,Vietnamese": 0.1875, "Indonesian,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Spanish": 0.2556818181818182, "English,Filipino,Chinese": 0.09659090909090909, "English,Filipino,Malay": 0.2159090909090909, "English,Filipino,Vietnamese": 0.13636363636363635, "English,Spanish,Chinese": 0.1590909090909091, "English,Spanish,Malay": 0.32954545454545453, "English,Spanish,Vietnamese": 0.23863636363636365, "English,Chinese,Malay": 0.10795454545454546, "English,Chinese,Vietnamese": 0.10795454545454546, "English,Malay,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Chinese": 0.11931818181818182, "Filipino,Spanish,Malay": 0.26136363636363635, "Filipino,Spanish,Vietnamese": 0.17613636363636365, "Filipino,Chinese,Malay": 0.17045454545454544, "Filipino,Chinese,Vietnamese": 0.16477272727272727, "Filipino,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay": 0.14772727272727273, "Spanish,Chinese,Vietnamese": 0.14204545454545456, "Spanish,Malay,Vietnamese": 0.19886363636363635, "Chinese,Malay,Vietnamese": 0.2215909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.19318181818181818, "Indonesian,English,Filipino,Chinese": 0.08522727272727272, "Indonesian,English,Filipino,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese": 0.11363636363636363, "Indonesian,English,Spanish,Malay": 0.24431818181818182, "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,English,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Malay": 0.19318181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay": 0.13636363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay": 0.10795454545454546, "Indonesian,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese": 0.07386363636363637, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, "English,Filipino,Malay,Vietnamese": 0.08522727272727272, "English,Spanish,Chinese,Malay": 0.09659090909090909, "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, "English,Spanish,Malay,Vietnamese": 0.13636363636363635, "English,Chinese,Malay,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.06818181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 } }, "AC3_2": 0.3893568250062849, "AC3_3": 0.26256352338772004, "AC3_4": 0.17448882958721418, "AC3_5": 0.1174343966908109, "AC3_6": 0.08242834749510006, "AC3_7": 0.06253362021073237 }, "prompt_2": { "overall_acc": 0.3758116883116883, "language_acc": { "Indonesian": 0.4034090909090909, "English": 0.6022727272727273, "Filipino": 0.29545454545454547, "Spanish": 0.4602272727272727, "Chinese": 0.24431818181818182, "Malay": 0.3352272727272727, "Vietnamese": 0.2897727272727273 }, "consistency_score_2": 0.4066558441558442, "consistency_score_3": 0.20844155844155843, "consistency_score_4": 0.11866883116883116, "consistency_score_5": 0.07196969696969698, "consistency_score_6": 0.04707792207792209, "consistency_score_7": 0.03409090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4715909090909091, "Indonesian,Filipino": 0.4772727272727273, "Indonesian,Spanish": 0.5454545454545454, "Indonesian,Chinese": 0.3352272727272727, "Indonesian,Malay": 0.5454545454545454, "Indonesian,Vietnamese": 0.3352272727272727, "English,Filipino": 0.3409090909090909, "English,Spanish": 0.6590909090909091, "English,Chinese": 0.2215909090909091, "English,Malay": 0.4147727272727273, "English,Vietnamese": 0.2784090909090909, "Filipino,Spanish": 0.4318181818181818, "Filipino,Chinese": 0.3068181818181818, "Filipino,Malay": 0.5113636363636364, "Filipino,Vietnamese": 0.3693181818181818, "Spanish,Chinese": 0.2556818181818182, "Spanish,Malay": 0.48295454545454547, "Spanish,Vietnamese": 0.32386363636363635, "Chinese,Malay": 0.35795454545454547, "Chinese,Vietnamese": 0.4943181818181818, "Malay,Vietnamese": 0.3806818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.25, "Indonesian,English,Spanish": 0.3806818181818182, "Indonesian,English,Chinese": 0.14204545454545456, "Indonesian,English,Malay": 0.2897727272727273, "Indonesian,English,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish": 0.3068181818181818, "Indonesian,Filipino,Chinese": 0.18181818181818182, "Indonesian,Filipino,Malay": 0.3409090909090909, "Indonesian,Filipino,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese": 0.16477272727272727, "Indonesian,Spanish,Malay": 0.3465909090909091, "Indonesian,Spanish,Vietnamese": 0.1875, "Indonesian,Chinese,Malay": 0.21022727272727273, "Indonesian,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Malay,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish": 0.26136363636363635, "English,Filipino,Chinese": 0.09659090909090909, "English,Filipino,Malay": 0.2215909090909091, "English,Filipino,Vietnamese": 0.125, "English,Spanish,Chinese": 0.1590909090909091, "English,Spanish,Malay": 0.3352272727272727, "English,Spanish,Vietnamese": 0.2159090909090909, "English,Chinese,Malay": 0.11363636363636363, "English,Chinese,Vietnamese": 0.10795454545454546, "English,Malay,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Chinese": 0.14204545454545456, "Filipino,Spanish,Malay": 0.2840909090909091, "Filipino,Spanish,Vietnamese": 0.17613636363636365, "Filipino,Chinese,Malay": 0.19318181818181818, "Filipino,Chinese,Vietnamese": 0.19318181818181818, "Filipino,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay": 0.16477272727272727, "Spanish,Chinese,Vietnamese": 0.14204545454545456, "Spanish,Malay,Vietnamese": 0.19318181818181818, "Chinese,Malay,Vietnamese": 0.2215909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.21022727272727273, "Indonesian,English,Filipino,Chinese": 0.08522727272727272, "Indonesian,English,Filipino,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese": 0.11931818181818182, "Indonesian,English,Spanish,Malay": 0.26136363636363635, "Indonesian,English,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,English,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese": 0.10795454545454546, "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.125, "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.13068181818181818, "English,Filipino,Spanish,Chinese": 0.07954545454545454, "English,Filipino,Spanish,Malay": 0.17613636363636365, "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, "English,Filipino,Malay,Vietnamese": 0.07954545454545454, "English,Spanish,Chinese,Malay": 0.10227272727272728, "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, "English,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Chinese,Malay,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.07386363636363637, "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 } }, "AC3_2": 0.39062584195040106, "AC3_3": 0.26815349136405475, "AC3_4": 0.18037974005271648, "AC3_5": 0.12080472394995091, "AC3_6": 0.08367400351986029, "AC3_7": 0.06251125110986225 }, "prompt_3": { "overall_acc": 0.3741883116883117, "language_acc": { "Indonesian": 0.4034090909090909, "English": 0.6193181818181818, "Filipino": 0.29545454545454547, "Spanish": 0.4602272727272727, "Chinese": 0.22727272727272727, "Malay": 0.3352272727272727, "Vietnamese": 0.2784090909090909 }, "consistency_score_2": 0.4120670995670995, "consistency_score_3": 0.20941558441558442, "consistency_score_4": 0.11672077922077921, "consistency_score_5": 0.06899350649350648, "consistency_score_6": 0.04301948051948051, "consistency_score_7": 0.028409090909090908, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4715909090909091, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.5340909090909091, "Indonesian,Chinese": 0.3465909090909091, "Indonesian,Malay": 0.5397727272727273, "Indonesian,Vietnamese": 0.3352272727272727, "English,Filipino": 0.3352272727272727, "English,Spanish": 0.6590909090909091, "English,Chinese": 0.23295454545454544, "English,Malay": 0.42045454545454547, "English,Vietnamese": 0.2840909090909091, "Filipino,Spanish": 0.4090909090909091, "Filipino,Chinese": 0.3068181818181818, "Filipino,Malay": 0.5340909090909091, "Filipino,Vietnamese": 0.3806818181818182, "Spanish,Chinese": 0.2727272727272727, "Spanish,Malay": 0.48863636363636365, "Spanish,Vietnamese": 0.3125, "Chinese,Malay": 0.39204545454545453, "Chinese,Vietnamese": 0.5340909090909091, "Malay,Vietnamese": 0.39204545454545453 }, "3_combine": { "Indonesian,English,Filipino": 0.23863636363636365, "Indonesian,English,Spanish": 0.3806818181818182, "Indonesian,English,Chinese": 0.14204545454545456, "Indonesian,English,Malay": 0.2897727272727273, "Indonesian,English,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish": 0.2897727272727273, "Indonesian,Filipino,Chinese": 0.18181818181818182, "Indonesian,Filipino,Malay": 0.3522727272727273, "Indonesian,Filipino,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese": 0.17045454545454544, "Indonesian,Spanish,Malay": 0.3465909090909091, "Indonesian,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,Chinese,Malay": 0.22727272727272727, "Indonesian,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,Malay,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish": 0.25, "English,Filipino,Chinese": 0.09090909090909091, "English,Filipino,Malay": 0.23295454545454544, "English,Filipino,Vietnamese": 0.11931818181818182, "English,Spanish,Chinese": 0.16477272727272727, "English,Spanish,Malay": 0.3352272727272727, "English,Spanish,Vietnamese": 0.21022727272727273, "English,Chinese,Malay": 0.125, "English,Chinese,Vietnamese": 0.11363636363636363, "English,Malay,Vietnamese": 0.13636363636363635, "Filipino,Spanish,Chinese": 0.13068181818181818, "Filipino,Spanish,Malay": 0.2840909090909091, "Filipino,Spanish,Vietnamese": 0.1590909090909091, "Filipino,Chinese,Malay": 0.20454545454545456, "Filipino,Chinese,Vietnamese": 0.21022727272727273, "Filipino,Malay,Vietnamese": 0.23295454545454544, "Spanish,Chinese,Malay": 0.17613636363636365, "Spanish,Chinese,Vietnamese": 0.1534090909090909, "Spanish,Malay,Vietnamese": 0.18181818181818182, "Chinese,Malay,Vietnamese": 0.24431818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.19886363636363635, "Indonesian,English,Filipino,Chinese": 0.07954545454545454, "Indonesian,English,Filipino,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese": 0.11931818181818182, "Indonesian,English,Spanish,Malay": 0.26136363636363635, "Indonesian,English,Spanish,Vietnamese": 0.125, "Indonesian,English,Chinese,Malay": 0.09659090909090909, "Indonesian,English,Chinese,Vietnamese": 0.0625, "Indonesian,English,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay": 0.125, "Indonesian,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,Spanish,Malay,Vietnamese": 0.125, "Indonesian,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese": 0.07386363636363637, "English,Filipino,Spanish,Malay": 0.18181818181818182, "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, "English,Filipino,Malay,Vietnamese": 0.07386363636363637, "English,Spanish,Chinese,Malay": 0.10795454545454546, "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, "English,Spanish,Malay,Vietnamese": 0.11931818181818182, "English,Chinese,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.06818181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 } }, "AC3_2": 0.39221527778996373, "AC3_3": 0.2685412639750828, "AC3_4": 0.17793743124939904, "AC3_5": 0.11650551826494414, "AC3_6": 0.07716723935691445, "AC3_7": 0.052808834297733406 }, "prompt_4": { "overall_acc": 0.3774350649350649, "language_acc": { "Indonesian": 0.4090909090909091, "English": 0.6022727272727273, "Filipino": 0.30113636363636365, "Spanish": 0.45454545454545453, "Chinese": 0.23295454545454544, "Malay": 0.3352272727272727, "Vietnamese": 0.3068181818181818 }, "consistency_score_2": 0.40882034632034636, "consistency_score_3": 0.2073051948051948, "consistency_score_4": 0.11542207792207793, "consistency_score_5": 0.06845238095238095, "consistency_score_6": 0.04301948051948052, "consistency_score_7": 0.028409090909090908, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4659090909090909, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.5397727272727273, "Indonesian,Chinese": 0.3352272727272727, "Indonesian,Malay": 0.5397727272727273, "Indonesian,Vietnamese": 0.35795454545454547, "English,Filipino": 0.3465909090909091, "English,Spanish": 0.6420454545454546, "English,Chinese": 0.2159090909090909, "English,Malay": 0.42613636363636365, "English,Vietnamese": 0.2897727272727273, "Filipino,Spanish": 0.4147727272727273, "Filipino,Chinese": 0.29545454545454547, "Filipino,Malay": 0.5284090909090909, "Filipino,Vietnamese": 0.36363636363636365, "Spanish,Chinese": 0.25, "Spanish,Malay": 0.4715909090909091, "Spanish,Vietnamese": 0.3409090909090909, "Chinese,Malay": 0.375, "Chinese,Vietnamese": 0.5113636363636364, "Malay,Vietnamese": 0.4034090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.23863636363636365, "Indonesian,English,Spanish": 0.3693181818181818, "Indonesian,English,Chinese": 0.13068181818181818, "Indonesian,English,Malay": 0.29545454545454547, "Indonesian,English,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish": 0.2840909090909091, "Indonesian,Filipino,Chinese": 0.17045454545454544, "Indonesian,Filipino,Malay": 0.3522727272727273, "Indonesian,Filipino,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese": 0.1590909090909091, "Indonesian,Spanish,Malay": 0.3409090909090909, "Indonesian,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,Chinese,Malay": 0.2215909090909091, "Indonesian,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,Malay,Vietnamese": 0.22727272727272727, "English,Filipino,Spanish": 0.25, "English,Filipino,Chinese": 0.08522727272727272, "English,Filipino,Malay": 0.24431818181818182, "English,Filipino,Vietnamese": 0.11931818181818182, "English,Spanish,Chinese": 0.14772727272727273, "English,Spanish,Malay": 0.32954545454545453, "English,Spanish,Vietnamese": 0.22727272727272727, "English,Chinese,Malay": 0.11363636363636363, "English,Chinese,Vietnamese": 0.10795454545454546, "English,Malay,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Chinese": 0.11931818181818182, "Filipino,Spanish,Malay": 0.2784090909090909, "Filipino,Spanish,Vietnamese": 0.1590909090909091, "Filipino,Chinese,Malay": 0.19318181818181818, "Filipino,Chinese,Vietnamese": 0.1875, "Filipino,Malay,Vietnamese": 0.22727272727272727, "Spanish,Chinese,Malay": 0.1534090909090909, "Spanish,Chinese,Vietnamese": 0.1534090909090909, "Spanish,Malay,Vietnamese": 0.19318181818181818, "Chinese,Malay,Vietnamese": 0.24431818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.19318181818181818, "Indonesian,English,Filipino,Chinese": 0.07386363636363637, "Indonesian,English,Filipino,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese": 0.10227272727272728, "Indonesian,English,Spanish,Malay": 0.2556818181818182, "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, "Indonesian,English,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Malay": 0.2159090909090909, "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese": 0.06818181818181818, "English,Filipino,Spanish,Malay": 0.17613636363636365, "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Malay,Vietnamese": 0.07386363636363637, "English,Spanish,Chinese,Malay": 0.09659090909090909, "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, "English,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Chinese,Malay,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Chinese,Malay": 0.09090909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.0625, "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 } }, "AC3_2": 0.392501295969958, "AC3_3": 0.2676205318376739, "AC3_4": 0.17678282682106855, "AC3_5": 0.11588722258149935, "AC3_6": 0.07723574686023443, "AC3_7": 0.052840909077889096 }, "prompt_5": { "overall_acc": 0.3628246753246754, "language_acc": { "Indonesian": 0.4034090909090909, "English": 0.6022727272727273, "Filipino": 0.26704545454545453, "Spanish": 0.42045454545454547, "Chinese": 0.2215909090909091, "Malay": 0.3409090909090909, "Vietnamese": 0.2840909090909091 }, "consistency_score_2": 0.38906926406926406, "consistency_score_3": 0.1912337662337662, "consistency_score_4": 0.10357142857142858, "consistency_score_5": 0.059253246753246745, "consistency_score_6": 0.03571428571428571, "consistency_score_7": 0.022727272727272728, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4772727272727273, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.5681818181818182, "Indonesian,Chinese": 0.24431818181818182, "Indonesian,Malay": 0.5511363636363636, "Indonesian,Vietnamese": 0.32954545454545453, "English,Filipino": 0.3352272727272727, "English,Spanish": 0.6477272727272727, "English,Chinese": 0.2727272727272727, "English,Malay": 0.4147727272727273, "English,Vietnamese": 0.2840909090909091, "Filipino,Spanish": 0.4090909090909091, "Filipino,Chinese": 0.26704545454545453, "Filipino,Malay": 0.4772727272727273, "Filipino,Vietnamese": 0.3806818181818182, "Spanish,Chinese": 0.2727272727272727, "Spanish,Malay": 0.5113636363636364, "Spanish,Vietnamese": 0.3181818181818182, "Chinese,Malay": 0.2784090909090909, "Chinese,Vietnamese": 0.24431818181818182, "Malay,Vietnamese": 0.4147727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.23295454545454544, "Indonesian,English,Spanish": 0.3806818181818182, "Indonesian,English,Chinese": 0.1534090909090909, "Indonesian,English,Malay": 0.30113636363636365, "Indonesian,English,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish": 0.29545454545454547, "Indonesian,Filipino,Chinese": 0.125, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese": 0.17613636363636365, "Indonesian,Spanish,Malay": 0.375, "Indonesian,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,Chinese,Malay": 0.1590909090909091, "Indonesian,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Malay,Vietnamese": 0.23295454545454544, "English,Filipino,Spanish": 0.2556818181818182, "English,Filipino,Chinese": 0.10227272727272728, "English,Filipino,Malay": 0.20454545454545456, "English,Filipino,Vietnamese": 0.11931818181818182, "English,Spanish,Chinese": 0.1875, "English,Spanish,Malay": 0.3409090909090909, "English,Spanish,Vietnamese": 0.20454545454545456, "English,Chinese,Malay": 0.10795454545454546, "English,Chinese,Vietnamese": 0.08522727272727272, "English,Malay,Vietnamese": 0.14204545454545456, "Filipino,Spanish,Chinese": 0.11931818181818182, "Filipino,Spanish,Malay": 0.26136363636363635, "Filipino,Spanish,Vietnamese": 0.1534090909090909, "Filipino,Chinese,Malay": 0.13068181818181818, "Filipino,Chinese,Vietnamese": 0.09090909090909091, "Filipino,Malay,Vietnamese": 0.2159090909090909, "Spanish,Chinese,Malay": 0.14772727272727273, "Spanish,Chinese,Vietnamese": 0.08522727272727272, "Spanish,Malay,Vietnamese": 0.20454545454545456, "Chinese,Malay,Vietnamese": 0.10795454545454546 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.19318181818181818, "Indonesian,English,Filipino,Chinese": 0.07954545454545454, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese": 0.14204545454545456, "Indonesian,English,Spanish,Malay": 0.26704545454545453, "Indonesian,English,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,English,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Malay": 0.21022727272727273, "Indonesian,Filipino,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay": 0.09659090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay": 0.10795454545454546, "Indonesian,Spanish,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.06818181818181818, "English,Filipino,Spanish,Chinese": 0.08522727272727272, "English,Filipino,Spanish,Malay": 0.16477272727272727, "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay": 0.056818181818181816, "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, "English,Filipino,Malay,Vietnamese": 0.06818181818181818, "English,Spanish,Chinese,Malay": 0.09659090909090909, "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, "English,Spanish,Malay,Vietnamese": 0.125, "English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Chinese,Malay": 0.07386363636363637, "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, "Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.07386363636363637, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay": 0.05113636363636364, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.0625, "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 } }, "AC3_2": 0.3754889406603043, "AC3_3": 0.25045852180212497, "AC3_4": 0.1611431554524962, "AC3_5": 0.10187000497086962, "AC3_6": 0.06502764036773935, "AC3_7": 0.04277511960613037 } }, "sg_eval": { "prompt_1": { "accuracy": 0.5436893203883495 }, "prompt_2": { "accuracy": 0.5728155339805825 }, "prompt_3": { "accuracy": 0.5631067961165048 }, "prompt_4": { "accuracy": 0.5631067961165048 }, "prompt_5": { "accuracy": 0.5631067961165048 } }, "cn_eval": { "prompt_1": { "accuracy": 0.23809523809523808 }, "prompt_2": { "accuracy": 0.23809523809523808 }, "prompt_3": { "accuracy": 0.23809523809523808 }, "prompt_4": { "accuracy": 0.23809523809523808 }, "prompt_5": { "accuracy": 0.23809523809523808 } }, "us_eval": { "prompt_1": { "accuracy": 0.6448598130841121 }, "prompt_2": { "accuracy": 0.6355140186915887 }, "prompt_3": { "accuracy": 0.6542056074766355 }, "prompt_4": { "accuracy": 0.6355140186915887 }, "prompt_5": { "accuracy": 0.6542056074766355 } }, "ph_eval": { "prompt_1": { "accuracy": 0.33, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.3333333333333333, "literature": 0.3, "politics": 0.3, "culture": 0.4, "film": 0.3, "law": 0.3, "geography": 0.4 } }, "prompt_2": { "accuracy": 0.3, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.4, "history": 0.4, "literature": 0.3, "politics": 0.3, "culture": 0.3, "film": 0.1, "law": 0.3, "geography": 0.4 } }, "prompt_3": { "accuracy": 0.32, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.2, "literature": 0.3, "politics": 0.3, "culture": 0.3, "film": 0.3, "law": 0.3, "geography": 0.6 } }, "prompt_4": { "accuracy": 0.33, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.5, "history": 0.2, "literature": 0.3, "politics": 0.3, "culture": 0.5, "film": 0.2, "law": 0.3, "geography": 0.5 } }, "prompt_5": { "accuracy": 0.31, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.3, "culture": 0.3, "film": 0.2, "law": 0.3, "geography": 0.5 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.17111295690470238 }, "prompt_2": { "bleu_score": 0.17578262863245286 }, "prompt_3": { "bleu_score": 0.181061792829869 }, "prompt_4": { "bleu_score": 0.17795860280445558 }, "prompt_5": { "bleu_score": 0.17554492731604188 } }, "indommlu": { "prompt_1": { "accuracy": 0.34328059282996193, "category_acc": { "History": 0.3293172690763052, "Geography": 0.2693877551020408, "Lampungic": 0.35374149659863946, "Social science": 0.4340567612687813, "Balinese": 0.3099787685774947, "Makassarese": 0.3279569892473118, "Banjarese": 0.2569444444444444, "Chemistry": 0.23941605839416058, "Biology": 0.30177514792899407, "Science": 0.35810113519091846, "Christian religion": 0.4079601990049751, "Art": 0.34608985024958405, "Islam religion": 0.3627311522048364, "Hindu religion": 0.29333333333333333, "Madurese": 0.3050847457627119, "Sport": 0.40540540540540543, "Indonesian language": 0.39632627646326274, "Physics": 0.30707070707070705, "Minangkabau culture": 0.2562814070351759, "Dayak language": 0.27522935779816515, "Sociology": 0.3548387096774194, "Economy": 0.33401639344262296, "Sundanese": 0.33189282627484873, "Javanese": 0.28024193548387094, "Civic education": 0.3977110157367668 } }, "prompt_2": { "accuracy": 0.33994258628746915, "category_acc": { "History": 0.3253012048192771, "Geography": 0.2714285714285714, "Lampungic": 0.36054421768707484, "Social science": 0.44240400667779634, "Balinese": 0.3036093418259023, "Makassarese": 0.2903225806451613, "Banjarese": 0.2569444444444444, "Chemistry": 0.25547445255474455, "Biology": 0.30414201183431955, "Science": 0.35500515995872034, "Christian religion": 0.39800995024875624, "Art": 0.3410981697171381, "Islam religion": 0.35704125177809387, "Hindu religion": 0.31333333333333335, "Madurese": 0.2847457627118644, "Sport": 0.3716216216216216, "Indonesian language": 0.39352428393524286, "Physics": 0.301010101010101, "Minangkabau culture": 0.2663316582914573, "Dayak language": 0.26605504587155965, "Sociology": 0.3528225806451613, "Economy": 0.32991803278688525, "Sundanese": 0.31892826274848746, "Javanese": 0.27318548387096775, "Civic education": 0.3948497854077253 } }, "prompt_3": { "accuracy": 0.3363375392215769, "category_acc": { "History": 0.321285140562249, "Geography": 0.24897959183673468, "Lampungic": 0.35374149659863946, "Social science": 0.44574290484140233, "Balinese": 0.3205944798301486, "Makassarese": 0.3118279569892473, "Banjarese": 0.2569444444444444, "Chemistry": 0.24087591240875914, "Biology": 0.3076923076923077, "Science": 0.33539731682146545, "Christian religion": 0.4129353233830846, "Art": 0.33777038269550747, "Islam religion": 0.34992887624466573, "Hindu religion": 0.30666666666666664, "Madurese": 0.2847457627118644, "Sport": 0.35135135135135137, "Indonesian language": 0.3894769613947696, "Physics": 0.28484848484848485, "Minangkabau culture": 0.271356783919598, "Dayak language": 0.27522935779816515, "Sociology": 0.3487903225806452, "Economy": 0.32991803278688525, "Sundanese": 0.32238547968885045, "Javanese": 0.2721774193548387, "Civic education": 0.39198855507868385 } }, "prompt_4": { "accuracy": 0.3396755457640697, "category_acc": { "History": 0.3273092369477912, "Geography": 0.25510204081632654, "Lampungic": 0.3333333333333333, "Social science": 0.44741235392320533, "Balinese": 0.3227176220806794, "Makassarese": 0.3118279569892473, "Banjarese": 0.2569444444444444, "Chemistry": 0.23795620437956205, "Biology": 0.3136094674556213, "Science": 0.34674922600619196, "Christian religion": 0.40298507462686567, "Art": 0.34775374376039936, "Islam religion": 0.3613086770981508, "Hindu religion": 0.32, "Madurese": 0.29491525423728815, "Sport": 0.3783783783783784, "Indonesian language": 0.3894769613947696, "Physics": 0.28888888888888886, "Minangkabau culture": 0.2663316582914573, "Dayak language": 0.25688073394495414, "Sociology": 0.3487903225806452, "Economy": 0.33401639344262296, "Sundanese": 0.33016421780466726, "Javanese": 0.2721774193548387, "Civic education": 0.39198855507868385 } }, "prompt_5": { "accuracy": 0.3384738634087723, "category_acc": { "History": 0.3232931726907631, "Geography": 0.2795918367346939, "Lampungic": 0.2925170068027211, "Social science": 0.4323873121869783, "Balinese": 0.3227176220806794, "Makassarese": 0.2903225806451613, "Banjarese": 0.2569444444444444, "Chemistry": 0.24233576642335766, "Biology": 0.293491124260355, "Science": 0.34881320949432404, "Christian religion": 0.3880597014925373, "Art": 0.3527454242928453, "Islam religion": 0.35135135135135137, "Hindu religion": 0.3333333333333333, "Madurese": 0.3050847457627119, "Sport": 0.3716216216216216, "Indonesian language": 0.3891656288916563, "Physics": 0.29494949494949496, "Minangkabau culture": 0.2663316582914573, "Dayak language": 0.26605504587155965, "Sociology": 0.3487903225806452, "Economy": 0.32786885245901637, "Sundanese": 0.3353500432152118, "Javanese": 0.27923387096774194, "Civic education": 0.38197424892703863 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.298729720423162 }, "prompt_2": { "bleu_score": 0.29714403066664635 }, "prompt_3": { "bleu_score": 0.29688089941244467 }, "prompt_4": { "bleu_score": 0.29657094327603384 }, "prompt_5": { "bleu_score": 0.2891515350076281 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.06332150290827988 }, "prompt_2": { "bleu_score": 0.06237270430229427 }, "prompt_3": { "bleu_score": 0.06261245548230009 }, "prompt_4": { "bleu_score": 0.0638050751721597 }, "prompt_5": { "bleu_score": 0.06116577273766147 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.015709981480553 }, "prompt_2": { "bleu_score": 0.015737669889031625 }, "prompt_3": { "bleu_score": 0.018161111996093608 }, "prompt_4": { "bleu_score": 0.015298302651452203 }, "prompt_5": { "bleu_score": 0.01808485717283886 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.27795819511180947 }, "prompt_2": { "bleu_score": 0.28022199889007265 }, "prompt_3": { "bleu_score": 0.2797980730259455 }, "prompt_4": { "bleu_score": 0.2800502337161776 }, "prompt_5": { "bleu_score": 0.26872088931620774 } }, "mmlu": { "prompt_1": { "accuracy": 0.5542590431738623 }, "prompt_2": { "accuracy": 0.5484247374562428 }, "prompt_3": { "accuracy": 0.543757292882147 }, "prompt_4": { "accuracy": 0.543757292882147 }, "prompt_5": { "accuracy": 0.5460910151691949 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.5319270647121916, "category_acc": { "high_school_european_history": 0.7439024390243902, "business_ethics": 0.6363636363636364, "clinical_knowledge": 0.6060606060606061, "medical_genetics": 0.5858585858585859, "high_school_us_history": 0.7635467980295566, "high_school_physics": 0.24666666666666667, "high_school_world_history": 0.7330508474576272, "virology": 0.4666666666666667, "high_school_microeconomics": 0.5864978902953587, "econometrics": 0.2831858407079646, "college_computer_science": 0.46464646464646464, "high_school_biology": 0.6699029126213593, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.398576512455516, "philosophy": 0.5548387096774193, "professional_medicine": 0.5166051660516605, "nutrition": 0.5770491803278689, "global_facts": 0.2828282828282828, "machine_learning": 0.36036036036036034, "security_studies": 0.6557377049180327, "public_relations": 0.6422018348623854, "professional_psychology": 0.5450081833060556, "prehistory": 0.5727554179566563, "anatomy": 0.4552238805970149, "human_sexuality": 0.5615384615384615, "college_medicine": 0.5523255813953488, "high_school_government_and_politics": 0.7864583333333334, "college_chemistry": 0.41414141414141414, "logical_fallacies": 0.654320987654321, "high_school_geography": 0.6802030456852792, "elementary_mathematics": 0.376657824933687, "human_aging": 0.5495495495495496, "college_mathematics": 0.35353535353535354, "high_school_psychology": 0.7830882352941176, "formal_logic": 0.392, "high_school_statistics": 0.3953488372093023, "international_law": 0.6666666666666666, "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.48484848484848486, "conceptual_physics": 0.4444444444444444, "miscellaneous": 0.7186700767263428, "high_school_chemistry": 0.3613861386138614, "marketing": 0.8154506437768241, "professional_law": 0.40378343118069143, "management": 0.7549019607843137, "college_physics": 0.38613861386138615, "jurisprudence": 0.6728971962616822, "world_religions": 0.6823529411764706, "sociology": 0.7, "us_foreign_policy": 0.7070707070707071, "high_school_macroeconomics": 0.5578406169665809, "computer_security": 0.6565656565656566, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6376811594202898, "electrical_engineering": 0.5069444444444444, "astronomy": 0.5364238410596026, "college_biology": 0.5104895104895105 } }, "prompt_2": { "accuracy": 0.5332856632105828, "category_acc": { "high_school_european_history": 0.75, "business_ethics": 0.6262626262626263, "clinical_knowledge": 0.6060606060606061, "medical_genetics": 0.5757575757575758, "high_school_us_history": 0.7684729064039408, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.7330508474576272, "virology": 0.4727272727272727, "high_school_microeconomics": 0.6118143459915611, "econometrics": 0.2831858407079646, "college_computer_science": 0.41414141414141414, "high_school_biology": 0.6634304207119741, "abstract_algebra": 0.20202020202020202, "professional_accounting": 0.4128113879003559, "philosophy": 0.5838709677419355, "professional_medicine": 0.5239852398523985, "nutrition": 0.5606557377049181, "global_facts": 0.30303030303030304, "machine_learning": 0.36036036036036034, "security_studies": 0.6557377049180327, "public_relations": 0.6055045871559633, "professional_psychology": 0.5368248772504092, "prehistory": 0.5727554179566563, "anatomy": 0.4253731343283582, "human_sexuality": 0.6, "college_medicine": 0.5523255813953488, "high_school_government_and_politics": 0.78125, "college_chemistry": 0.40404040404040403, "logical_fallacies": 0.6666666666666666, "high_school_geography": 0.6802030456852792, "elementary_mathematics": 0.38461538461538464, "human_aging": 0.536036036036036, "college_mathematics": 0.35353535353535354, "high_school_psychology": 0.7775735294117647, "formal_logic": 0.376, "high_school_statistics": 0.4186046511627907, "international_law": 0.675, "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.494949494949495, "conceptual_physics": 0.41452991452991456, "miscellaneous": 0.7225063938618926, "high_school_chemistry": 0.36633663366336633, "marketing": 0.8154506437768241, "professional_law": 0.4050880626223092, "management": 0.7647058823529411, "college_physics": 0.38613861386138615, "jurisprudence": 0.6915887850467289, "world_religions": 0.6764705882352942, "sociology": 0.695, "us_foreign_policy": 0.7171717171717171, "high_school_macroeconomics": 0.5732647814910026, "computer_security": 0.6666666666666666, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6405797101449275, "electrical_engineering": 0.5347222222222222, "astronomy": 0.5629139072847682, "college_biology": 0.48951048951048953 } }, "prompt_3": { "accuracy": 0.5327851269217018, "category_acc": { "high_school_european_history": 0.7439024390243902, "business_ethics": 0.6363636363636364, "clinical_knowledge": 0.5909090909090909, "medical_genetics": 0.5757575757575758, "high_school_us_history": 0.7635467980295566, "high_school_physics": 0.26, "high_school_world_history": 0.7288135593220338, "virology": 0.4727272727272727, "high_school_microeconomics": 0.6075949367088608, "econometrics": 0.25663716814159293, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.6666666666666666, "abstract_algebra": 0.21212121212121213, "professional_accounting": 0.398576512455516, "philosophy": 0.5870967741935483, "professional_medicine": 0.5239852398523985, "nutrition": 0.5737704918032787, "global_facts": 0.29292929292929293, "machine_learning": 0.34234234234234234, "security_studies": 0.6680327868852459, "public_relations": 0.6238532110091743, "professional_psychology": 0.5384615384615384, "prehistory": 0.5603715170278638, "anatomy": 0.43283582089552236, "human_sexuality": 0.6, "college_medicine": 0.5465116279069767, "high_school_government_and_politics": 0.796875, "college_chemistry": 0.41414141414141414, "logical_fallacies": 0.6728395061728395, "high_school_geography": 0.7157360406091371, "elementary_mathematics": 0.35013262599469497, "human_aging": 0.5630630630630631, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.7904411764705882, "formal_logic": 0.384, "high_school_statistics": 0.3767441860465116, "international_law": 0.6666666666666666, "high_school_mathematics": 0.2788104089219331, "high_school_computer_science": 0.42424242424242425, "conceptual_physics": 0.4230769230769231, "miscellaneous": 0.7225063938618926, "high_school_chemistry": 0.35148514851485146, "marketing": 0.8412017167381974, "professional_law": 0.4044357469015003, "management": 0.7549019607843137, "college_physics": 0.40594059405940597, "jurisprudence": 0.6822429906542056, "world_religions": 0.6823529411764706, "sociology": 0.705, "us_foreign_policy": 0.7373737373737373, "high_school_macroeconomics": 0.5681233933161953, "computer_security": 0.6565656565656566, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6492753623188405, "electrical_engineering": 0.5486111111111112, "astronomy": 0.5761589403973509, "college_biology": 0.4755244755244755 } }, "prompt_4": { "accuracy": 0.5308544869503039, "category_acc": { "high_school_european_history": 0.7560975609756098, "business_ethics": 0.6363636363636364, "clinical_knowledge": 0.5909090909090909, "medical_genetics": 0.5858585858585859, "high_school_us_history": 0.7586206896551724, "high_school_physics": 0.24, "high_school_world_history": 0.7245762711864406, "virology": 0.4727272727272727, "high_school_microeconomics": 0.5991561181434599, "econometrics": 0.25663716814159293, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.6699029126213593, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.398576512455516, "philosophy": 0.567741935483871, "professional_medicine": 0.5202952029520295, "nutrition": 0.5672131147540984, "global_facts": 0.31313131313131315, "machine_learning": 0.35135135135135137, "security_studies": 0.6680327868852459, "public_relations": 0.6146788990825688, "professional_psychology": 0.5482815057283142, "prehistory": 0.56656346749226, "anatomy": 0.44029850746268656, "human_sexuality": 0.6, "college_medicine": 0.5406976744186046, "high_school_government_and_politics": 0.7708333333333334, "college_chemistry": 0.37373737373737376, "logical_fallacies": 0.6604938271604939, "high_school_geography": 0.7258883248730964, "elementary_mathematics": 0.3209549071618037, "human_aging": 0.5585585585585585, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.7830882352941176, "formal_logic": 0.392, "high_school_statistics": 0.4, "international_law": 0.6583333333333333, "high_school_mathematics": 0.2788104089219331, "high_school_computer_science": 0.45454545454545453, "conceptual_physics": 0.42735042735042733, "miscellaneous": 0.7237851662404092, "high_school_chemistry": 0.3465346534653465, "marketing": 0.8326180257510729, "professional_law": 0.39856490541422046, "management": 0.7647058823529411, "college_physics": 0.40594059405940597, "jurisprudence": 0.6635514018691588, "world_religions": 0.7176470588235294, "sociology": 0.695, "us_foreign_policy": 0.7575757575757576, "high_school_macroeconomics": 0.570694087403599, "computer_security": 0.6363636363636364, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6405797101449275, "electrical_engineering": 0.5763888888888888, "astronomy": 0.5761589403973509, "college_biology": 0.46853146853146854 } }, "prompt_5": { "accuracy": 0.5337146943153379, "category_acc": { "high_school_european_history": 0.7560975609756098, "business_ethics": 0.6565656565656566, "clinical_knowledge": 0.5757575757575758, "medical_genetics": 0.5353535353535354, "high_school_us_history": 0.7733990147783252, "high_school_physics": 0.23333333333333334, "high_school_world_history": 0.7330508474576272, "virology": 0.4727272727272727, "high_school_microeconomics": 0.5907172995780591, "econometrics": 0.2743362831858407, "college_computer_science": 0.43434343434343436, "high_school_biology": 0.6634304207119741, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.4092526690391459, "philosophy": 0.5774193548387097, "professional_medicine": 0.5166051660516605, "nutrition": 0.5868852459016394, "global_facts": 0.29292929292929293, "machine_learning": 0.34234234234234234, "security_studies": 0.680327868852459, "public_relations": 0.6055045871559633, "professional_psychology": 0.5417348608837971, "prehistory": 0.5572755417956656, "anatomy": 0.4253731343283582, "human_sexuality": 0.6153846153846154, "college_medicine": 0.5406976744186046, "high_school_government_and_politics": 0.7864583333333334, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.6604938271604939, "high_school_geography": 0.6954314720812182, "elementary_mathematics": 0.34748010610079577, "human_aging": 0.5495495495495496, "college_mathematics": 0.32323232323232326, "high_school_psychology": 0.7830882352941176, "formal_logic": 0.4, "high_school_statistics": 0.413953488372093, "international_law": 0.6583333333333333, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.45454545454545453, "conceptual_physics": 0.4358974358974359, "miscellaneous": 0.7276214833759591, "high_school_chemistry": 0.37623762376237624, "marketing": 0.8283261802575107, "professional_law": 0.4050880626223092, "management": 0.7745098039215687, "college_physics": 0.40594059405940597, "jurisprudence": 0.6915887850467289, "world_religions": 0.6941176470588235, "sociology": 0.705, "us_foreign_policy": 0.7272727272727273, "high_school_macroeconomics": 0.5784061696658098, "computer_security": 0.6464646464646465, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6579710144927536, "electrical_engineering": 0.5486111111111112, "astronomy": 0.5695364238410596, "college_biology": 0.4965034965034965 } } }, "c_eval": { "prompt_1": { "accuracy": 0.24888558692421991 }, "prompt_2": { "accuracy": 0.24888558692421991 }, "prompt_3": { "accuracy": 0.24888558692421991 }, "prompt_4": { "accuracy": 0.24591381872213966 }, "prompt_5": { "accuracy": 0.24888558692421991 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.23848069738480698, "category_acc": { "computer_network": 0.125, "operating_system": 0.125, "computer_architecture": 0.23076923076923078, "college_programming": 0.21428571428571427, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.125, "middle_school_chemistry": 0.16, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.36666666666666664, "business_administration": 0.18421052631578946, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.2647058823529412, "teacher_qualification": 0.2653061224489796, "high_school_politics": 0.25, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.17857142857142858, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.18518518518518517, "law": 0.27586206896551724, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.2, "middle_school_history": 0.18518518518518517, "civil_servant": 0.25, "sports_science": 0.08333333333333333, "plant_protection": 0.18518518518518517, "basic_medicine": 0.125, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.16666666666666666, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.2222222222222222, "physician": 0.2777777777777778 } }, "prompt_2": { "accuracy": 0.23910336239103364, "category_acc": { "computer_network": 0.125, "operating_system": 0.125, "computer_architecture": 0.23076923076923078, "college_programming": 0.23809523809523808, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.125, "middle_school_chemistry": 0.16, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.36666666666666664, "business_administration": 0.18421052631578946, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.2647058823529412, "teacher_qualification": 0.2653061224489796, "high_school_politics": 0.25, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.17857142857142858, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.18518518518518517, "law": 0.27586206896551724, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.2, "middle_school_history": 0.18518518518518517, "civil_servant": 0.25, "sports_science": 0.08333333333333333, "plant_protection": 0.18518518518518517, "basic_medicine": 0.125, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.16666666666666666, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.2222222222222222, "physician": 0.2777777777777778 } }, "prompt_3": { "accuracy": 0.23848069738480698, "category_acc": { "computer_network": 0.125, "operating_system": 0.125, "computer_architecture": 0.23076923076923078, "college_programming": 0.21428571428571427, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.125, "middle_school_chemistry": 0.16, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.36666666666666664, "business_administration": 0.18421052631578946, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.2647058823529412, "teacher_qualification": 0.2653061224489796, "high_school_politics": 0.25, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.17857142857142858, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.18518518518518517, "law": 0.27586206896551724, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.2, "middle_school_history": 0.18518518518518517, "civil_servant": 0.25, "sports_science": 0.08333333333333333, "plant_protection": 0.18518518518518517, "basic_medicine": 0.125, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.16666666666666666, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.2222222222222222, "physician": 0.2777777777777778 } }, "prompt_4": { "accuracy": 0.24159402241594022, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.08333333333333333, "computer_architecture": 0.3076923076923077, "college_programming": 0.2619047619047619, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.25, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.13793103448275862, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.08333333333333333, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.31666666666666665, "business_administration": 0.18421052631578946, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.23529411764705882, "teacher_qualification": 0.2857142857142857, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.18518518518518517, "law": 0.3103448275862069, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.25, "high_school_chinese": 0.25, "high_school_history": 0.2, "middle_school_history": 0.18518518518518517, "civil_servant": 0.25, "sports_science": 0.08333333333333333, "plant_protection": 0.2222222222222222, "basic_medicine": 0.125, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.18518518518518517, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.25925925925925924, "physician": 0.2777777777777778 } }, "prompt_5": { "accuracy": 0.23910336239103364, "category_acc": { "computer_network": 0.125, "operating_system": 0.125, "computer_architecture": 0.23076923076923078, "college_programming": 0.23809523809523808, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.125, "middle_school_chemistry": 0.16, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.36666666666666664, "business_administration": 0.18421052631578946, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.2647058823529412, "teacher_qualification": 0.2653061224489796, "high_school_politics": 0.25, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.23076923076923078, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.17857142857142858, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.18518518518518517, "law": 0.27586206896551724, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.2, "middle_school_history": 0.18518518518518517, "civil_servant": 0.25, "sports_science": 0.08333333333333333, "plant_protection": 0.18518518518518517, "basic_medicine": 0.125, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.16666666666666666, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.2222222222222222, "physician": 0.2777777777777778 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.27956989247311825 }, "prompt_2": { "accuracy": 0.2831541218637993 }, "prompt_3": { "accuracy": 0.27598566308243727 }, "prompt_4": { "accuracy": 0.2831541218637993 }, "prompt_5": { "accuracy": 0.2831541218637993 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2526333966499741, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.25675675675675674, "ancient_chinese": 0.2682926829268293, "arts": 0.25625, "astronomy": 0.21818181818181817, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.25190839694656486, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.24458204334365324, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.27358490566037735, "college_education": 0.308411214953271, "college_engineering_hydrology": 0.29245283018867924, "college_law": 0.2222222222222222, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2490842490842491, "computer_science": 0.2696078431372549, "computer_security": 0.24561403508771928, "conceptual_physics": 0.23809523809523808, "construction_project_management": 0.2446043165467626, "economics": 0.2389937106918239, "education": 0.25153374233128833, "electrical_engineering": 0.23255813953488372, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.25252525252525254, "elementary_information_and_technology": 0.2773109243697479, "elementary_mathematics": 0.2565217391304348, "ethnology": 0.24444444444444444, "food_science": 0.24475524475524477, "genetics": 0.26704545454545453, "global_facts": 0.2483221476510067, "high_school_biology": 0.21893491124260356, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.2818181818181818, "high_school_politics": 0.2727272727272727, "human_sexuality": 0.25396825396825395, "international_law": 0.25405405405405407, "journalism": 0.2558139534883721, "jurisprudence": 0.24817518248175183, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.2682926829268293, "machine_learning": 0.2459016393442623, "management": 0.24761904761904763, "marketing": 0.24444444444444444, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.27586206896551724, "nutrition": 0.23448275862068965, "philosophy": 0.2571428571428571, "professional_accounting": 0.25142857142857145, "professional_law": 0.25118483412322273, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.24568965517241378, "public_relations": 0.2471264367816092, "security_study": 0.25925925925925924, "sociology": 0.252212389380531, "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.2485207100591716, "world_history": 0.2360248447204969, "world_religions": 0.26875 } }, "prompt_2": { "accuracy": 0.2526333966499741, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.2635135135135135, "ancient_chinese": 0.2682926829268293, "arts": 0.25625, "astronomy": 0.21212121212121213, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.25190839694656486, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.24458204334365324, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.27358490566037735, "college_education": 0.308411214953271, "college_engineering_hydrology": 0.29245283018867924, "college_law": 0.2222222222222222, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2490842490842491, "computer_science": 0.27450980392156865, "computer_security": 0.24561403508771928, "conceptual_physics": 0.23809523809523808, "construction_project_management": 0.2446043165467626, "economics": 0.2389937106918239, "education": 0.25153374233128833, "electrical_engineering": 0.23837209302325582, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.23737373737373738, "elementary_information_and_technology": 0.2773109243697479, "elementary_mathematics": 0.2565217391304348, "ethnology": 0.24444444444444444, "food_science": 0.23776223776223776, "genetics": 0.2727272727272727, "global_facts": 0.2483221476510067, "high_school_biology": 0.21893491124260356, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.2727272727272727, "high_school_politics": 0.2727272727272727, "human_sexuality": 0.25396825396825395, "international_law": 0.25405405405405407, "journalism": 0.25, "jurisprudence": 0.24817518248175183, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.2682926829268293, "machine_learning": 0.2540983606557377, "management": 0.24761904761904763, "marketing": 0.24444444444444444, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.27586206896551724, "nutrition": 0.2413793103448276, "philosophy": 0.2571428571428571, "professional_accounting": 0.25142857142857145, "professional_law": 0.25118483412322273, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.24568965517241378, "public_relations": 0.2471264367816092, "security_study": 0.25925925925925924, "sociology": 0.252212389380531, "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.2485207100591716, "world_history": 0.2360248447204969, "world_religions": 0.26875 } }, "prompt_3": { "accuracy": 0.2528060783975134, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.2635135135135135, "ancient_chinese": 0.2621951219512195, "arts": 0.25625, "astronomy": 0.23030303030303031, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.25190839694656486, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.24458204334365324, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.2830188679245283, "college_education": 0.308411214953271, "college_engineering_hydrology": 0.29245283018867924, "college_law": 0.2222222222222222, "college_mathematics": 0.23809523809523808, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2490842490842491, "computer_science": 0.2696078431372549, "computer_security": 0.24561403508771928, "conceptual_physics": 0.23809523809523808, "construction_project_management": 0.2446043165467626, "economics": 0.2389937106918239, "education": 0.25153374233128833, "electrical_engineering": 0.23255813953488372, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.24242424242424243, "elementary_information_and_technology": 0.2773109243697479, "elementary_mathematics": 0.2608695652173913, "ethnology": 0.24444444444444444, "food_science": 0.23776223776223776, "genetics": 0.26704545454545453, "global_facts": 0.24161073825503357, "high_school_biology": 0.21893491124260356, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.2727272727272727, "high_school_politics": 0.2727272727272727, "human_sexuality": 0.25396825396825395, "international_law": 0.25405405405405407, "journalism": 0.25, "jurisprudence": 0.24817518248175183, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.2682926829268293, "machine_learning": 0.26229508196721313, "management": 0.24761904761904763, "marketing": 0.24444444444444444, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.27586206896551724, "nutrition": 0.2413793103448276, "philosophy": 0.2571428571428571, "professional_accounting": 0.25142857142857145, "professional_law": 0.25118483412322273, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.24568965517241378, "public_relations": 0.2471264367816092, "security_study": 0.26666666666666666, "sociology": 0.252212389380531, "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.2485207100591716, "world_history": 0.2360248447204969, "world_religions": 0.26875 } }, "prompt_4": { "accuracy": 0.2530651010188223, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.2635135135135135, "ancient_chinese": 0.25609756097560976, "arts": 0.25625, "astronomy": 0.24242424242424243, "business_ethics": 0.24401913875598086, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.25190839694656486, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.25077399380804954, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.2641509433962264, "college_education": 0.29906542056074764, "college_engineering_hydrology": 0.2830188679245283, "college_law": 0.21296296296296297, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.2641509433962264, "college_medicine": 0.2454212454212454, "computer_science": 0.2647058823529412, "computer_security": 0.24561403508771928, "conceptual_physics": 0.25170068027210885, "construction_project_management": 0.2517985611510791, "economics": 0.2389937106918239, "education": 0.25153374233128833, "electrical_engineering": 0.23837209302325582, "elementary_chinese": 0.2777777777777778, "elementary_commonsense": 0.25757575757575757, "elementary_information_and_technology": 0.29831932773109243, "elementary_mathematics": 0.27391304347826084, "ethnology": 0.24444444444444444, "food_science": 0.2517482517482518, "genetics": 0.23863636363636365, "global_facts": 0.26174496644295303, "high_school_biology": 0.23668639053254437, "high_school_chemistry": 0.25, "high_school_geography": 0.2627118644067797, "high_school_mathematics": 0.27439024390243905, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.2727272727272727, "human_sexuality": 0.25396825396825395, "international_law": 0.2594594594594595, "journalism": 0.25, "jurisprudence": 0.25060827250608275, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.2601626016260163, "machine_learning": 0.21311475409836064, "management": 0.24761904761904763, "marketing": 0.25, "marxist_theory": 0.24338624338624337, "modern_chinese": 0.25862068965517243, "nutrition": 0.2413793103448276, "philosophy": 0.23809523809523808, "professional_accounting": 0.25142857142857145, "professional_law": 0.26066350710900477, "professional_medicine": 0.24202127659574468, "professional_psychology": 0.24568965517241378, "public_relations": 0.2471264367816092, "security_study": 0.25925925925925924, "sociology": 0.252212389380531, "sports_science": 0.26666666666666666, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.2485207100591716, "world_history": 0.2422360248447205, "world_religions": 0.26875 } }, "prompt_5": { "accuracy": 0.25297876014505266, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.2635135135135135, "ancient_chinese": 0.2682926829268293, "arts": 0.25625, "astronomy": 0.21818181818181817, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.25190839694656486, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.24458204334365324, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.2830188679245283, "college_education": 0.308411214953271, "college_engineering_hydrology": 0.29245283018867924, "college_law": 0.2222222222222222, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.25274725274725274, "computer_science": 0.27450980392156865, "computer_security": 0.24561403508771928, "conceptual_physics": 0.23809523809523808, "construction_project_management": 0.2446043165467626, "economics": 0.2389937106918239, "education": 0.25153374233128833, "electrical_engineering": 0.23837209302325582, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.24242424242424243, "elementary_information_and_technology": 0.2773109243697479, "elementary_mathematics": 0.2565217391304348, "ethnology": 0.24444444444444444, "food_science": 0.23776223776223776, "genetics": 0.2727272727272727, "global_facts": 0.2483221476510067, "high_school_biology": 0.21893491124260356, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.2727272727272727, "high_school_politics": 0.27972027972027974, "human_sexuality": 0.25396825396825395, "international_law": 0.25405405405405407, "journalism": 0.25, "jurisprudence": 0.24817518248175183, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.2682926829268293, "machine_learning": 0.2459016393442623, "management": 0.24761904761904763, "marketing": 0.24444444444444444, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.27586206896551724, "nutrition": 0.2413793103448276, "philosophy": 0.2571428571428571, "professional_accounting": 0.25142857142857145, "professional_law": 0.25118483412322273, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.24568965517241378, "public_relations": 0.2471264367816092, "security_study": 0.25925925925925924, "sociology": 0.25663716814159293, "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.2485207100591716, "world_history": 0.2360248447204969, "world_religions": 0.26875 } } }, "zbench": { "prompt_1": { "accuracy": 0.18181818181818182 }, "prompt_2": { "accuracy": 0.18181818181818182 }, "prompt_3": { "accuracy": 0.18181818181818182 }, "prompt_4": { "accuracy": 0.15151515151515152 }, "prompt_5": { "accuracy": 0.18181818181818182 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.47045454545454546 }, "prompt_3": { "accuracy": 0.4863636363636364 }, "prompt_4": { "accuracy": 0.41363636363636364 }, "prompt_5": { "accuracy": 0.46136363636363636 } }, "ocnli": { "prompt_1": { "accuracy": 0.34372881355932206 }, "prompt_2": { "accuracy": 0.34372881355932206 }, "prompt_3": { "accuracy": 0.34440677966101696 }, "prompt_4": { "accuracy": 0.3440677966101695 }, "prompt_5": { "accuracy": 0.3440677966101695 } }, "c3": { "prompt_1": { "accuracy": 0.26626776364996263 }, "prompt_2": { "accuracy": 0.26626776364996263 }, "prompt_3": { "accuracy": 0.2666417352281227 }, "prompt_4": { "accuracy": 0.26514584891548243 }, "prompt_5": { "accuracy": 0.2673896783844428 } }, "dream": { "prompt_1": { "accuracy": 0.9363057324840764 }, "prompt_2": { "accuracy": 0.932876041156296 }, "prompt_3": { "accuracy": 0.9353258206761391 }, "prompt_4": { "accuracy": 0.9372856442920137 }, "prompt_5": { "accuracy": 0.9338559529642332 } }, "samsum": { "prompt_1": { "rouge1": 0.5392068754143731, "rouge2": 0.2950280260068732, "rougeL": 0.4506368895080014, "avg_rouge": 0.4282905969764159 }, "prompt_2": { "rouge1": 0.5359945124407804, "rouge2": 0.29276027430028645, "rougeL": 0.44971572354652717, "avg_rouge": 0.4261568367625314 }, "prompt_3": { "rouge1": 0.5369216789882587, "rouge2": 0.2932230928399554, "rougeL": 0.4508405893944396, "avg_rouge": 0.4269951204075513 }, "prompt_4": { "rouge1": 0.538447184048574, "rouge2": 0.29528202636477136, "rougeL": 0.4516316980089774, "avg_rouge": 0.42845363614077425 }, "prompt_5": { "rouge1": 0.5365832097389195, "rouge2": 0.2954036639870932, "rougeL": 0.45167227299325957, "avg_rouge": 0.42788638223975745 } }, "dialogsum": { "prompt_1": { "rouge1": 0.23098825839708775, "rouge2": 0.06235340689809025, "rougeL": 0.1710693119069414, "avg_rouge": 0.15480365906737314 }, "prompt_2": { "rouge1": 0.23057479562213193, "rouge2": 0.0626605453865275, "rougeL": 0.16987328159127832, "avg_rouge": 0.15436954086664592 }, "prompt_3": { "rouge1": 0.23134441905824937, "rouge2": 0.06285173990336988, "rougeL": 0.17136941481315845, "avg_rouge": 0.15518852459159257 }, "prompt_4": { "rouge1": 0.23060303444666233, "rouge2": 0.06339967025872248, "rougeL": 0.16971858763937736, "avg_rouge": 0.15457376411492071 }, "prompt_5": { "rouge1": 0.23634127906884056, "rouge2": 0.06468803693978939, "rougeL": 0.17715576683843468, "avg_rouge": 0.15939502761568822 } }, "sst2": { "prompt_1": { "accuracy": 0.9529816513761468 }, "prompt_2": { "accuracy": 0.9506880733944955 }, "prompt_3": { "accuracy": 0.9564220183486238 }, "prompt_4": { "accuracy": 0.9518348623853211 }, "prompt_5": { "accuracy": 0.9575688073394495 } }, "cola": { "prompt_1": { "accuracy": 0.8235858101629914 }, "prompt_2": { "accuracy": 0.8235858101629914 }, "prompt_3": { "accuracy": 0.8245445829338447 }, "prompt_4": { "accuracy": 0.8216682646212847 }, "prompt_5": { "accuracy": 0.8235858101629914 } }, "qqp": { "prompt_1": { "accuracy": 0.8415 }, "prompt_2": { "accuracy": 0.8435 }, "prompt_3": { "accuracy": 0.838 }, "prompt_4": { "accuracy": 0.849 }, "prompt_5": { "accuracy": 0.843 } }, "mnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } }, "qnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "wnli": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.6 } }, "rte": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "flan_ul2": { "model_size": "20B", "model_link": "https://huggingface.co/google/flan-t5-ul2", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.7878151260504201, "language_acc": { "Spanish": 0.9621848739495799, "English": 0.9781512605042016, "Chinese": 0.3663865546218487, "Vietnamese": 0.8445378151260504 }, "consistency_score_2": 0.6305322128851542, "consistency_score_3": 0.46554621848739497, "consistency_score_4": 0.3302521008403361, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9663865546218487, "Spanish,Chinese": 0.37058823529411766, "Spanish,Vietnamese": 0.8394957983193277, "English,Chinese": 0.3722689075630252, "English,Vietnamese": 0.8436974789915966, "Chinese,Vietnamese": 0.3907563025210084 }, "3_combine": { "Spanish,English,Chinese": 0.3621848739495798, "Spanish,English,Vietnamese": 0.8285714285714286, "Spanish,Chinese,Vietnamese": 0.3352941176470588, "English,Chinese,Vietnamese": 0.33613445378151263 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3302521008403361 } }, "AC3_2": 0.7004529865170198, "AC3_3": 0.5852491851117807, "AC3_4": 0.4654060045248004 }, "prompt_2": { "overall_acc": 0.7903361344537815, "language_acc": { "Spanish": 0.9621848739495799, "English": 0.9773109243697479, "Chinese": 0.373109243697479, "Vietnamese": 0.8487394957983193 }, "consistency_score_2": 0.634733893557423, "consistency_score_3": 0.47016806722689086, "consistency_score_4": 0.33613445378151263, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9672268907563025, "Spanish,Chinese": 0.3773109243697479, "Spanish,Vietnamese": 0.8411764705882353, "English,Chinese": 0.3773109243697479, "English,Vietnamese": 0.846218487394958, "Chinese,Vietnamese": 0.39915966386554624 }, "3_combine": { "Spanish,English,Chinese": 0.36722689075630255, "Spanish,English,Vietnamese": 0.83109243697479, "Spanish,Chinese,Vietnamese": 0.3411764705882353, "English,Chinese,Vietnamese": 0.3411764705882353 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.33613445378151263 } }, "AC3_2": 0.7040399726964512, "AC3_3": 0.589590756255747, "AC3_4": 0.47166647333599176 }, "prompt_3": { "overall_acc": 0.792016806722689, "language_acc": { "Spanish": 0.9621848739495799, "English": 0.9773109243697479, "Chinese": 0.37563025210084033, "Vietnamese": 0.8529411764705882 }, "consistency_score_2": 0.6372549019607844, "consistency_score_3": 0.4756302521008404, "consistency_score_4": 0.3420168067226891, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.965546218487395, "Spanish,Chinese": 0.3815126050420168, "Spanish,Vietnamese": 0.8487394957983193, "English,Chinese": 0.37815126050420167, "English,Vietnamese": 0.8521008403361344, "Chinese,Vietnamese": 0.39747899159663863 }, "3_combine": { "Spanish,English,Chinese": 0.3697478991596639, "Spanish,English,Vietnamese": 0.8369747899159664, "Spanish,Chinese,Vietnamese": 0.3487394957983193, "English,Chinese,Vietnamese": 0.34705882352941175 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3420168067226891 } }, "AC3_2": 0.7062570250536211, "AC3_3": 0.594340752495431, "AC3_4": 0.4777337388776721 }, "prompt_4": { "overall_acc": 0.7913865546218487, "language_acc": { "Spanish": 0.9630252100840336, "English": 0.9773109243697479, "Chinese": 0.3764705882352941, "Vietnamese": 0.8487394957983193 }, "consistency_score_2": 0.6355742296918768, "consistency_score_3": 0.4726890756302522, "consistency_score_4": 0.33949579831932775, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9663865546218487, "Spanish,Chinese": 0.3815126050420168, "Spanish,Vietnamese": 0.8436974789915966, "English,Chinese": 0.37815126050420167, "English,Vietnamese": 0.8478991596638655, "Chinese,Vietnamese": 0.3957983193277311 }, "3_combine": { "Spanish,English,Chinese": 0.3697478991596639, "Spanish,English,Vietnamese": 0.8327731092436975, "Spanish,Chinese,Vietnamese": 0.3445378151260504, "English,Chinese,Vietnamese": 0.34369747899159664 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.33949579831932775 } }, "AC3_2": 0.7049736829999066, "AC3_3": 0.5918629708353986, "AC3_4": 0.47515536769675665 }, "prompt_5": { "overall_acc": 0.7922268907563025, "language_acc": { "Spanish": 0.9613445378151261, "English": 0.9773109243697479, "Chinese": 0.37899159663865545, "Vietnamese": 0.8512605042016806 }, "consistency_score_2": 0.6378151260504202, "consistency_score_3": 0.47478991596638664, "consistency_score_4": 0.3403361344537815, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9672268907563025, "Spanish,Chinese": 0.38571428571428573, "Spanish,Vietnamese": 0.8445378151260504, "English,Chinese": 0.3831932773109244, "English,Vietnamese": 0.8487394957983193, "Chinese,Vietnamese": 0.39747899159663863 }, "3_combine": { "Spanish,English,Chinese": 0.3739495798319328, "Spanish,English,Vietnamese": 0.8336134453781513, "Spanish,Chinese,Vietnamese": 0.346218487394958, "English,Chinese,Vietnamese": 0.3453781512605042 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.3403361344537815 } }, "AC3_2": 0.7066845424321856, "AC3_3": 0.5937432508598683, "AC3_4": 0.47612968388495314 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.3914285714285714, "language_acc": { "Filipino": 0.36, "Vietnamese": 0.30666666666666664, "Chinese": 0.3, "Spanish": 0.41333333333333333, "Malay": 0.36, "Indonesian": 0.4066666666666667, "English": 0.5933333333333334 }, "consistency_score_2": 0.4546031746031745, "consistency_score_3": 0.2617142857142857, "consistency_score_4": 0.16952380952380952, "consistency_score_5": 0.11936507936507937, "consistency_score_6": 0.09047619047619049, "consistency_score_7": 0.07333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4, "Filipino,Chinese": 0.36666666666666664, "Filipino,Spanish": 0.44, "Filipino,Malay": 0.43333333333333335, "Filipino,Indonesian": 0.5066666666666667, "Filipino,English": 0.5, "Vietnamese,Chinese": 0.4266666666666667, "Vietnamese,Spanish": 0.3933333333333333, "Vietnamese,Malay": 0.4, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.37333333333333335, "Chinese,Spanish": 0.42, "Chinese,Malay": 0.36666666666666664, "Chinese,Indonesian": 0.41333333333333333, "Chinese,English": 0.3933333333333333, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.6466666666666666, "Malay,Indonesian": 0.5933333333333334, "Malay,English": 0.4666666666666667, "Indonesian,English": 0.5333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.20666666666666667, "Filipino,Vietnamese,Spanish": 0.21333333333333335, "Filipino,Vietnamese,Malay": 0.21333333333333335, "Filipino,Vietnamese,Indonesian": 0.26, "Filipino,Vietnamese,English": 0.23333333333333334, "Filipino,Chinese,Spanish": 0.21333333333333335, "Filipino,Chinese,Malay": 0.16666666666666666, "Filipino,Chinese,Indonesian": 0.22, "Filipino,Chinese,English": 0.20666666666666667, "Filipino,Spanish,Malay": 0.2733333333333333, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.3333333333333333, "Filipino,Malay,English": 0.2866666666666667, "Filipino,Indonesian,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish": 0.23333333333333334, "Vietnamese,Chinese,Malay": 0.20666666666666667, "Vietnamese,Chinese,Indonesian": 0.26, "Vietnamese,Chinese,English": 0.2, "Vietnamese,Spanish,Malay": 0.21333333333333335, "Vietnamese,Spanish,Indonesian": 0.26, "Vietnamese,Spanish,English": 0.2733333333333333, "Vietnamese,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Malay,English": 0.22, "Vietnamese,Indonesian,English": 0.26666666666666666, "Chinese,Spanish,Malay": 0.22666666666666666, "Chinese,Spanish,Indonesian": 0.23333333333333334, "Chinese,Spanish,English": 0.29333333333333333, "Chinese,Malay,Indonesian": 0.24666666666666667, "Chinese,Malay,English": 0.22666666666666666, "Chinese,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian": 0.35333333333333333, "Spanish,Malay,English": 0.36, "Spanish,Indonesian,English": 0.37333333333333335, "Malay,Indonesian,English": 0.35333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Spanish,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Chinese,Malay,English": 0.12, "Filipino,Chinese,Indonesian,English": 0.14666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.22, "Filipino,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.14, "Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Vietnamese,Chinese,Spanish,English": 0.16, "Vietnamese,Chinese,Malay,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Malay,English": 0.14, "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Chinese,Spanish,Malay,Indonesian": 0.16, "Chinese,Spanish,Malay,English": 0.19333333333333333, "Chinese,Spanish,Indonesian,English": 0.18, "Chinese,Malay,Indonesian,English": 0.16666666666666666, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.12, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 } }, "AC3_2": 0.4206571964123532, "AC3_3": 0.31369078860339067, "AC3_4": 0.23658501087220715, "AC3_5": 0.18294237766106827, "AC3_6": 0.14697910781817347, "AC3_7": 0.1235245901373564 }, "prompt_2": { "overall_acc": 0.3885714285714285, "language_acc": { "Filipino": 0.36, "Vietnamese": 0.32, "Chinese": 0.29333333333333333, "Spanish": 0.4266666666666667, "Malay": 0.36, "Indonesian": 0.38666666666666666, "English": 0.5733333333333334 }, "consistency_score_2": 0.4565079365079365, "consistency_score_3": 0.26552380952380955, "consistency_score_4": 0.17542857142857146, "consistency_score_5": 0.12666666666666668, "consistency_score_6": 0.09809523809523811, "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.41333333333333333, "Filipino,Chinese": 0.3933333333333333, "Filipino,Spanish": 0.4266666666666667, "Filipino,Malay": 0.46, "Filipino,Indonesian": 0.4866666666666667, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.41333333333333333, "Vietnamese,Spanish": 0.4066666666666667, "Vietnamese,Malay": 0.4, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.38, "Chinese,Spanish": 0.4066666666666667, "Chinese,Malay": 0.44, "Chinese,Indonesian": 0.4666666666666667, "Chinese,English": 0.38, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.6133333333333333, "Malay,Indonesian": 0.6066666666666667, "Malay,English": 0.4533333333333333, "Indonesian,English": 0.47333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.21333333333333335, "Filipino,Vietnamese,Spanish": 0.22666666666666666, "Filipino,Vietnamese,Malay": 0.23333333333333334, "Filipino,Vietnamese,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,English": 0.22666666666666666, "Filipino,Chinese,Spanish": 0.20666666666666667, "Filipino,Chinese,Malay": 0.22, "Filipino,Chinese,Indonesian": 0.24666666666666667, "Filipino,Chinese,English": 0.22666666666666666, "Filipino,Spanish,Malay": 0.26666666666666666, "Filipino,Spanish,Indonesian": 0.3, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.3333333333333333, "Filipino,Malay,English": 0.29333333333333333, "Filipino,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish": 0.22666666666666666, "Vietnamese,Chinese,Malay": 0.24666666666666667, "Vietnamese,Chinese,Indonesian": 0.2733333333333333, "Vietnamese,Chinese,English": 0.2, "Vietnamese,Spanish,Malay": 0.22666666666666666, "Vietnamese,Spanish,Indonesian": 0.25333333333333335, "Vietnamese,Spanish,English": 0.26, "Vietnamese,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Malay,English": 0.22666666666666666, "Vietnamese,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay": 0.25333333333333335, "Chinese,Spanish,Indonesian": 0.24, "Chinese,Spanish,English": 0.28, "Chinese,Malay,Indonesian": 0.32666666666666666, "Chinese,Malay,English": 0.24666666666666667, "Chinese,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian": 0.3466666666666667, "Spanish,Malay,English": 0.34, "Spanish,Indonesian,English": 0.34, "Malay,Indonesian,English": 0.32666666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay": 0.16, "Filipino,Vietnamese,Chinese,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,English": 0.14, "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,English": 0.16, "Filipino,Vietnamese,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Malay,English": 0.16, "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Chinese,Spanish,English": 0.17333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.18, "Filipino,Chinese,Malay,English": 0.15333333333333332, "Filipino,Chinese,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Spanish,Malay,English": 0.22, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Vietnamese,Chinese,Spanish,English": 0.16, "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Malay,English": 0.16, "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Spanish,Malay,English": 0.16, "Vietnamese,Spanish,Indonesian,English": 0.16, "Vietnamese,Malay,Indonesian,English": 0.17333333333333334, "Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Chinese,Spanish,Malay,English": 0.18666666666666668, "Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Chinese,Malay,Indonesian,English": 0.19333333333333333, "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.12, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Chinese,Spanish,Malay,English": 0.12, "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, "Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, "AC3_2": 0.4198089513295224, "AC3_3": 0.31547383304936494, "AC3_4": 0.24172528581616537, "AC3_5": 0.19105360439914856, "AC3_6": 0.15664523340365272, "AC3_7": 0.1326829268009518 }, "prompt_3": { "overall_acc": 0.38095238095238093, "language_acc": { "Filipino": 0.32666666666666666, "Vietnamese": 0.3, "Chinese": 0.2733333333333333, "Spanish": 0.41333333333333333, "Malay": 0.36, "Indonesian": 0.3933333333333333, "English": 0.6 }, "consistency_score_2": 0.45968253968253964, "consistency_score_3": 0.2700952380952381, "consistency_score_4": 0.1771428571428571, "consistency_score_5": 0.12571428571428572, "consistency_score_6": 0.09619047619047619, "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.3933333333333333, "Filipino,Chinese": 0.3933333333333333, "Filipino,Spanish": 0.44, "Filipino,Malay": 0.48, "Filipino,Indonesian": 0.5466666666666666, "Filipino,English": 0.4866666666666667, "Vietnamese,Chinese": 0.3933333333333333, "Vietnamese,Spanish": 0.38666666666666666, "Vietnamese,Malay": 0.4066666666666667, "Vietnamese,Indonesian": 0.5, "Vietnamese,English": 0.36666666666666664, "Chinese,Spanish": 0.42, "Chinese,Malay": 0.38, "Chinese,Indonesian": 0.44666666666666666, "Chinese,English": 0.37333333333333335, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.5066666666666667, "Spanish,English": 0.6266666666666667, "Malay,Indonesian": 0.6266666666666667, "Malay,English": 0.4666666666666667, "Indonesian,English": 0.5266666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.20666666666666667, "Filipino,Vietnamese,Spanish": 0.21333333333333335, "Filipino,Vietnamese,Malay": 0.24, "Filipino,Vietnamese,Indonesian": 0.28, "Filipino,Vietnamese,English": 0.21333333333333335, "Filipino,Chinese,Spanish": 0.22, "Filipino,Chinese,Malay": 0.2, "Filipino,Chinese,Indonesian": 0.25333333333333335, "Filipino,Chinese,English": 0.21333333333333335, "Filipino,Spanish,Malay": 0.2866666666666667, "Filipino,Spanish,Indonesian": 0.3333333333333333, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.38, "Filipino,Malay,English": 0.3, "Filipino,Indonesian,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish": 0.21333333333333335, "Vietnamese,Chinese,Malay": 0.21333333333333335, "Vietnamese,Chinese,Indonesian": 0.2733333333333333, "Vietnamese,Chinese,English": 0.18666666666666668, "Vietnamese,Spanish,Malay": 0.22666666666666666, "Vietnamese,Spanish,Indonesian": 0.26, "Vietnamese,Spanish,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Malay,English": 0.22666666666666666, "Vietnamese,Indonesian,English": 0.26666666666666666, "Chinese,Spanish,Malay": 0.23333333333333334, "Chinese,Spanish,Indonesian": 0.24666666666666667, "Chinese,Spanish,English": 0.28, "Chinese,Malay,Indonesian": 0.2866666666666667, "Chinese,Malay,English": 0.22, "Chinese,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian": 0.38, "Spanish,Malay,English": 0.36, "Spanish,Indonesian,English": 0.38, "Malay,Indonesian,English": 0.37333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.14, "Filipino,Vietnamese,Chinese,Malay": 0.14, "Filipino,Vietnamese,Chinese,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,English": 0.12, "Filipino,Vietnamese,Spanish,Malay": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Malay,English": 0.16, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Chinese,Spanish,English": 0.17333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.16, "Filipino,Chinese,Malay,English": 0.14, "Filipino,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Spanish,Malay,English": 0.23333333333333334, "Filipino,Spanish,Indonesian,English": 0.26666666666666666, "Filipino,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,Malay,English": 0.14, "Vietnamese,Chinese,Indonesian,English": 0.14, "Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Spanish,Malay,English": 0.16, "Vietnamese,Spanish,Indonesian,English": 0.18, "Vietnamese,Malay,Indonesian,English": 0.2, "Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Chinese,Spanish,Malay,English": 0.18, "Chinese,Spanish,Indonesian,English": 0.18, "Chinese,Malay,Indonesian,English": 0.18666666666666668, "Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.12, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Indonesian,English": 0.14, "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, "AC3_2": 0.41663070056905455, "AC3_3": 0.3160857086644258, "AC3_4": 0.241833252028828, "AC3_5": 0.18904403863078753, "AC3_6": 0.15359756673902836, "AC3_7": 0.1322314049299911 }, "prompt_4": { "overall_acc": 0.3895238095238095, "language_acc": { "Filipino": 0.36666666666666664, "Vietnamese": 0.30666666666666664, "Chinese": 0.3, "Spanish": 0.41333333333333333, "Malay": 0.36666666666666664, "Indonesian": 0.4066666666666667, "English": 0.5666666666666667 }, "consistency_score_2": 0.4571428571428572, "consistency_score_3": 0.26742857142857146, "consistency_score_4": 0.17676190476190473, "consistency_score_5": 0.12666666666666665, "consistency_score_6": 0.09714285714285716, "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4066666666666667, "Filipino,Chinese": 0.4066666666666667, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.4666666666666667, "Filipino,Indonesian": 0.5066666666666667, "Filipino,English": 0.4866666666666667, "Vietnamese,Chinese": 0.4066666666666667, "Vietnamese,Spanish": 0.4066666666666667, "Vietnamese,Malay": 0.4066666666666667, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.35333333333333333, "Chinese,Spanish": 0.4266666666666667, "Chinese,Malay": 0.42, "Chinese,Indonesian": 0.44, "Chinese,English": 0.38, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.5, "Spanish,English": 0.6, "Malay,Indonesian": 0.6133333333333333, "Malay,English": 0.44666666666666666, "Indonesian,English": 0.49333333333333335 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2, "Filipino,Vietnamese,Spanish": 0.23333333333333334, "Filipino,Vietnamese,Malay": 0.23333333333333334, "Filipino,Vietnamese,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,English": 0.21333333333333335, "Filipino,Chinese,Spanish": 0.23333333333333334, "Filipino,Chinese,Malay": 0.21333333333333335, "Filipino,Chinese,Indonesian": 0.24, "Filipino,Chinese,English": 0.22666666666666666, "Filipino,Spanish,Malay": 0.2866666666666667, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.3466666666666667, "Filipino,Malay,English": 0.29333333333333333, "Filipino,Indonesian,English": 0.32, "Vietnamese,Chinese,Spanish": 0.24, "Vietnamese,Chinese,Malay": 0.23333333333333334, "Vietnamese,Chinese,Indonesian": 0.2733333333333333, "Vietnamese,Chinese,English": 0.19333333333333333, "Vietnamese,Spanish,Malay": 0.22666666666666666, "Vietnamese,Spanish,Indonesian": 0.26, "Vietnamese,Spanish,English": 0.24666666666666667, "Vietnamese,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Malay,English": 0.21333333333333335, "Vietnamese,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay": 0.26, "Chinese,Spanish,Indonesian": 0.24666666666666667, "Chinese,Spanish,English": 0.28, "Chinese,Malay,Indonesian": 0.31333333333333335, "Chinese,Malay,English": 0.24, "Chinese,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian": 0.36, "Spanish,Malay,English": 0.3333333333333333, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay": 0.14, "Filipino,Vietnamese,Chinese,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.18, "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay": 0.16666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Chinese,Spanish,English": 0.18, "Filipino,Chinese,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Malay,English": 0.14666666666666667, "Filipino,Chinese,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Spanish,Malay,English": 0.22666666666666666, "Filipino,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.16, "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Spanish,English": 0.15333333333333332, "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Malay,English": 0.14, "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Spanish,Malay,English": 0.15333333333333332, "Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Chinese,Spanish,Malay,English": 0.19333333333333333, "Chinese,Spanish,Indonesian,English": 0.18, "Chinese,Malay,Indonesian,English": 0.18666666666666668, "Spanish,Malay,Indonesian,English": 0.25333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,English": 0.12, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, "AC3_2": 0.42063313509414024, "AC3_3": 0.31713043113260764, "AC3_4": 0.24317396244484493, "AC3_5": 0.19116851164808235, "AC3_6": 0.15550461277207275, "AC3_7": 0.1327383366857251 }, "prompt_5": { "overall_acc": 0.3866666666666667, "language_acc": { "Filipino": 0.34, "Vietnamese": 0.32666666666666666, "Chinese": 0.28, "Spanish": 0.43333333333333335, "Malay": 0.35333333333333333, "Indonesian": 0.38666666666666666, "English": 0.5866666666666667 }, "consistency_score_2": 0.4523809523809524, "consistency_score_3": 0.26285714285714284, "consistency_score_4": 0.17314285714285715, "consistency_score_5": 0.12444444444444444, "consistency_score_6": 0.09619047619047619, "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4066666666666667, "Filipino,Chinese": 0.36, "Filipino,Spanish": 0.44666666666666666, "Filipino,Malay": 0.4533333333333333, "Filipino,Indonesian": 0.5466666666666666, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.4, "Vietnamese,Spanish": 0.4, "Vietnamese,Malay": 0.42, "Vietnamese,Indonesian": 0.4866666666666667, "Vietnamese,English": 0.35333333333333333, "Chinese,Spanish": 0.41333333333333333, "Chinese,Malay": 0.4, "Chinese,Indonesian": 0.4066666666666667, "Chinese,English": 0.3466666666666667, "Spanish,Malay": 0.5133333333333333, "Spanish,Indonesian": 0.4666666666666667, "Spanish,English": 0.6266666666666667, "Malay,Indonesian": 0.6266666666666667, "Malay,English": 0.4533333333333333, "Indonesian,English": 0.48 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2, "Filipino,Vietnamese,Spanish": 0.22, "Filipino,Vietnamese,Malay": 0.23333333333333334, "Filipino,Vietnamese,Indonesian": 0.28, "Filipino,Vietnamese,English": 0.21333333333333335, "Filipino,Chinese,Spanish": 0.2, "Filipino,Chinese,Malay": 0.18, "Filipino,Chinese,Indonesian": 0.22666666666666666, "Filipino,Chinese,English": 0.19333333333333333, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.36666666666666664, "Filipino,Malay,English": 0.29333333333333333, "Filipino,Indonesian,English": 0.32, "Vietnamese,Chinese,Spanish": 0.23333333333333334, "Vietnamese,Chinese,Malay": 0.24, "Vietnamese,Chinese,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,English": 0.18666666666666668, "Vietnamese,Spanish,Malay": 0.24666666666666667, "Vietnamese,Spanish,Indonesian": 0.26, "Vietnamese,Spanish,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Malay,English": 0.22666666666666666, "Vietnamese,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay": 0.24666666666666667, "Chinese,Spanish,Indonesian": 0.22, "Chinese,Spanish,English": 0.26666666666666666, "Chinese,Malay,Indonesian": 0.29333333333333333, "Chinese,Malay,English": 0.22, "Chinese,Indonesian,English": 0.19333333333333333, "Spanish,Malay,Indonesian": 0.36666666666666664, "Spanish,Malay,English": 0.36666666666666664, "Spanish,Indonesian,English": 0.34, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,English": 0.12, "Filipino,Vietnamese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,English": 0.15333333333333332, "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay": 0.14, "Filipino,Chinese,Spanish,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,English": 0.15333333333333332, "Filipino,Chinese,Malay,Indonesian": 0.16, "Filipino,Chinese,Malay,English": 0.12666666666666668, "Filipino,Chinese,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Spanish,Malay,English": 0.24, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Spanish,English": 0.15333333333333332, "Vietnamese,Chinese,Malay,Indonesian": 0.2, "Vietnamese,Chinese,Malay,English": 0.14666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.14, "Vietnamese,Spanish,Malay,Indonesian": 0.2, "Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Chinese,Spanish,Malay,English": 0.18, "Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Chinese,Malay,Indonesian,English": 0.17333333333333334, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Spanish,Malay,English": 0.12, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Chinese,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Vietnamese,Chinese,Spanish,Malay,English": 0.12, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, "AC3_2": 0.41695043506192236, "AC3_3": 0.3129618767846613, "AC3_4": 0.23918339567010166, "AC3_5": 0.18828985503562445, "AC3_6": 0.15405654171694425, "AC3_7": 0.1325714285430204 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.39123376623376627, "language_acc": { "Indonesian": 0.3465909090909091, "English": 0.6136363636363636, "Filipino": 0.375, "Spanish": 0.5, "Chinese": 0.25, "Malay": 0.3522727272727273, "Vietnamese": 0.30113636363636365 }, "consistency_score_2": 0.3825757575757576, "consistency_score_3": 0.18295454545454543, "consistency_score_4": 0.09756493506493508, "consistency_score_5": 0.053030303030303025, "consistency_score_6": 0.02516233766233766, "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4034090909090909, "Indonesian,Filipino": 0.44886363636363635, "Indonesian,Spanish": 0.5113636363636364, "Indonesian,Chinese": 0.24431818181818182, "Indonesian,Malay": 0.5795454545454546, "Indonesian,Vietnamese": 0.38636363636363635, "English,Filipino": 0.3465909090909091, "English,Spanish": 0.6534090909090909, "English,Chinese": 0.2556818181818182, "English,Malay": 0.3977272727272727, "English,Vietnamese": 0.2784090909090909, "Filipino,Spanish": 0.4431818181818182, "Filipino,Chinese": 0.22727272727272727, "Filipino,Malay": 0.48863636363636365, "Filipino,Vietnamese": 0.4943181818181818, "Spanish,Chinese": 0.22727272727272727, "Spanish,Malay": 0.42045454545454547, "Spanish,Vietnamese": 0.39204545454545453, "Chinese,Malay": 0.2215909090909091, "Chinese,Vietnamese": 0.19886363636363635, "Malay,Vietnamese": 0.4147727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.19318181818181818, "Indonesian,English,Spanish": 0.3352272727272727, "Indonesian,English,Chinese": 0.09090909090909091, "Indonesian,English,Malay": 0.2556818181818182, "Indonesian,English,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish": 0.26136363636363635, "Indonesian,Filipino,Chinese": 0.10795454545454546, "Indonesian,Filipino,Malay": 0.3068181818181818, "Indonesian,Filipino,Vietnamese": 0.23295454545454544, "Indonesian,Spanish,Chinese": 0.125, "Indonesian,Spanish,Malay": 0.29545454545454547, "Indonesian,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,Chinese,Malay": 0.1534090909090909, "Indonesian,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish": 0.2840909090909091, "English,Filipino,Chinese": 0.06818181818181818, "English,Filipino,Malay": 0.2215909090909091, "English,Filipino,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese": 0.14204545454545456, "English,Spanish,Malay": 0.29545454545454547, "English,Spanish,Vietnamese": 0.24431818181818182, "English,Chinese,Malay": 0.07954545454545454, "English,Chinese,Vietnamese": 0.056818181818181816, "English,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese": 0.10227272727272728, "Filipino,Spanish,Malay": 0.2556818181818182, "Filipino,Spanish,Vietnamese": 0.23863636363636365, "Filipino,Chinese,Malay": 0.11363636363636363, "Filipino,Chinese,Vietnamese": 0.11931818181818182, "Filipino,Malay,Vietnamese": 0.25, "Spanish,Chinese,Malay": 0.08522727272727272, "Spanish,Chinese,Vietnamese": 0.08522727272727272, "Spanish,Malay,Vietnamese": 0.2159090909090909, "Chinese,Malay,Vietnamese": 0.09659090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.17045454545454544, "Indonesian,English,Filipino,Chinese": 0.028409090909090908, "Indonesian,English,Filipino,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese": 0.06818181818181818, "Indonesian,English,Spanish,Malay": 0.2159090909090909, "Indonesian,English,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,English,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Chinese": 0.0625, "Indonesian,Filipino,Spanish,Malay": 0.1875, "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, "Indonesian,Filipino,Chinese,Vietnamese": 0.0625, "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Chinese": 0.05113636363636364, "English,Filipino,Spanish,Malay": 0.18181818181818182, "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay": 0.03977272727272727, "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.05113636363636364, "English,Spanish,Chinese,Vietnamese": 0.045454545454545456, "English,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Malay,Vietnamese": 0.125, "Indonesian,English,Chinese,Malay,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, "AC3_2": 0.3868563254426954, "AC3_3": 0.24931888858844456, "AC3_4": 0.1561816628750512, "AC3_5": 0.09340050933563672, "AC3_6": 0.04728361306173132, "AC3_7": 0.011200966719617114 }, "prompt_2": { "overall_acc": 0.39285714285714285, "language_acc": { "Indonesian": 0.36363636363636365, "English": 0.6022727272727273, "Filipino": 0.38636363636363635, "Spanish": 0.48863636363636365, "Chinese": 0.26136363636363635, "Malay": 0.3465909090909091, "Vietnamese": 0.30113636363636365 }, "consistency_score_2": 0.3831168831168831, "consistency_score_3": 0.17987012987012987, "consistency_score_4": 0.09334415584415583, "consistency_score_5": 0.04978354978354977, "consistency_score_6": 0.023538961038961033, "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.42045454545454547, "Indonesian,Filipino": 0.48295454545454547, "Indonesian,Spanish": 0.5, "Indonesian,Chinese": 0.23295454545454544, "Indonesian,Malay": 0.5852272727272727, "Indonesian,Vietnamese": 0.3977272727272727, "English,Filipino": 0.36363636363636365, "English,Spanish": 0.6477272727272727, "English,Chinese": 0.26136363636363635, "English,Malay": 0.3977272727272727, "English,Vietnamese": 0.30113636363636365, "Filipino,Spanish": 0.4431818181818182, "Filipino,Chinese": 0.2159090909090909, "Filipino,Malay": 0.45454545454545453, "Filipino,Vietnamese": 0.48295454545454547, "Spanish,Chinese": 0.25, "Spanish,Malay": 0.39204545454545453, "Spanish,Vietnamese": 0.375, "Chinese,Malay": 0.24431818181818182, "Chinese,Vietnamese": 0.18181818181818182, "Malay,Vietnamese": 0.4147727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.2159090909090909, "Indonesian,English,Spanish": 0.3352272727272727, "Indonesian,English,Chinese": 0.09090909090909091, "Indonesian,English,Malay": 0.2727272727272727, "Indonesian,English,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish": 0.26704545454545453, "Indonesian,Filipino,Chinese": 0.10227272727272728, "Indonesian,Filipino,Malay": 0.29545454545454547, "Indonesian,Filipino,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Chinese": 0.125, "Indonesian,Spanish,Malay": 0.2897727272727273, "Indonesian,Spanish,Vietnamese": 0.23295454545454544, "Indonesian,Chinese,Malay": 0.16477272727272727, "Indonesian,Chinese,Vietnamese": 0.07954545454545454, "Indonesian,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Spanish": 0.2897727272727273, "English,Filipino,Chinese": 0.07386363636363637, "English,Filipino,Malay": 0.21022727272727273, "English,Filipino,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese": 0.1534090909090909, "English,Spanish,Malay": 0.2784090909090909, "English,Spanish,Vietnamese": 0.23863636363636365, "English,Chinese,Malay": 0.08522727272727272, "English,Chinese,Vietnamese": 0.056818181818181816, "English,Malay,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Chinese": 0.10227272727272728, "Filipino,Spanish,Malay": 0.2215909090909091, "Filipino,Spanish,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay": 0.10795454545454546, "Filipino,Chinese,Vietnamese": 0.09659090909090909, "Filipino,Malay,Vietnamese": 0.24431818181818182, "Spanish,Chinese,Malay": 0.09659090909090909, "Spanish,Chinese,Vietnamese": 0.07386363636363637, "Spanish,Malay,Vietnamese": 0.19318181818181818, "Chinese,Malay,Vietnamese": 0.09659090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.17613636363636365, "Indonesian,English,Filipino,Chinese": 0.03409090909090909, "Indonesian,English,Filipino,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese": 0.07386363636363637, "Indonesian,English,Spanish,Malay": 0.2159090909090909, "Indonesian,English,Spanish,Vietnamese": 0.1534090909090909, "Indonesian,English,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.05113636363636364, "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay": 0.06818181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese": 0.0625, "English,Filipino,Spanish,Malay": 0.1590909090909091, "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay": 0.03977272727272727, "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.06818181818181818, "English,Spanish,Chinese,Vietnamese": 0.03977272727272727, "English,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.028409090909090908, "Indonesian,English,Filipino,Spanish,Malay": 0.125, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Chinese,Malay,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, "AC3_2": 0.38792588159973884, "AC3_3": 0.24676060896242177, "AC3_4": 0.15084664914617452, "AC3_5": 0.08836884385010232, "AC3_6": 0.04441659703750908, "AC3_7": 0.011201629325091566 }, "prompt_3": { "overall_acc": 0.39204545454545453, "language_acc": { "Indonesian": 0.3693181818181818, "English": 0.6079545454545454, "Filipino": 0.3806818181818182, "Spanish": 0.5113636363636364, "Chinese": 0.25, "Malay": 0.3352272727272727, "Vietnamese": 0.2897727272727273 }, "consistency_score_2": 0.39420995670995673, "consistency_score_3": 0.1943181818181819, "consistency_score_4": 0.10649350649350653, "consistency_score_5": 0.05979437229437229, "consistency_score_6": 0.030844155844155844, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.42613636363636365, "Indonesian,Filipino": 0.4659090909090909, "Indonesian,Spanish": 0.5056818181818182, "Indonesian,Chinese": 0.26704545454545453, "Indonesian,Malay": 0.6079545454545454, "Indonesian,Vietnamese": 0.4318181818181818, "English,Filipino": 0.3522727272727273, "English,Spanish": 0.6534090909090909, "English,Chinese": 0.26704545454545453, "English,Malay": 0.42613636363636365, "English,Vietnamese": 0.2784090909090909, "Filipino,Spanish": 0.42045454545454547, "Filipino,Chinese": 0.25, "Filipino,Malay": 0.4659090909090909, "Filipino,Vietnamese": 0.4715909090909091, "Spanish,Chinese": 0.2556818181818182, "Spanish,Malay": 0.4090909090909091, "Spanish,Vietnamese": 0.3977272727272727, "Chinese,Malay": 0.26704545454545453, "Chinese,Vietnamese": 0.23295454545454544, "Malay,Vietnamese": 0.42613636363636365 }, "3_combine": { "Indonesian,English,Filipino": 0.19886363636363635, "Indonesian,English,Spanish": 0.3465909090909091, "Indonesian,English,Chinese": 0.11931818181818182, "Indonesian,English,Malay": 0.2897727272727273, "Indonesian,English,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish": 0.25, "Indonesian,Filipino,Chinese": 0.13068181818181818, "Indonesian,Filipino,Malay": 0.3181818181818182, "Indonesian,Filipino,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Chinese": 0.14204545454545456, "Indonesian,Spanish,Malay": 0.30113636363636365, "Indonesian,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Chinese,Malay": 0.17045454545454544, "Indonesian,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Malay,Vietnamese": 0.29545454545454547, "English,Filipino,Spanish": 0.2784090909090909, "English,Filipino,Chinese": 0.08522727272727272, "English,Filipino,Malay": 0.2215909090909091, "English,Filipino,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese": 0.17045454545454544, "English,Spanish,Malay": 0.3068181818181818, "English,Spanish,Vietnamese": 0.24431818181818182, "English,Chinese,Malay": 0.11363636363636363, "English,Chinese,Vietnamese": 0.07386363636363637, "English,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese": 0.10795454545454546, "Filipino,Spanish,Malay": 0.23295454545454544, "Filipino,Spanish,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay": 0.11931818181818182, "Filipino,Chinese,Vietnamese": 0.125, "Filipino,Malay,Vietnamese": 0.2556818181818182, "Spanish,Chinese,Malay": 0.10227272727272728, "Spanish,Chinese,Vietnamese": 0.09659090909090909, "Spanish,Malay,Vietnamese": 0.2215909090909091, "Chinese,Malay,Vietnamese": 0.13636363636363635 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.16477272727272727, "Indonesian,English,Filipino,Chinese": 0.03977272727272727, "Indonesian,English,Filipino,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese": 0.09659090909090909, "Indonesian,English,Spanish,Malay": 0.23863636363636365, "Indonesian,English,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,English,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Chinese": 0.06818181818181818, "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, "Indonesian,Filipino,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese,Malay": 0.09090909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese": 0.06818181818181818, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, "English,Filipino,Chinese,Malay": 0.05113636363636364, "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.08522727272727272, "English,Spanish,Chinese,Vietnamese": 0.0625, "English,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Chinese,Malay,Vietnamese": 0.0625, "Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.125, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Spanish,Chinese,Malay": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0625, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "English,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, "AC3_2": 0.39312472622166406, "AC3_3": 0.25984408029395195, "AC3_4": 0.16749060119194473, "AC3_5": 0.10376292866511384, "AC3_6": 0.057188972242627374, "AC3_7": 0.022087067856240646 }, "prompt_4": { "overall_acc": 0.387987012987013, "language_acc": { "Indonesian": 0.3693181818181818, "English": 0.5965909090909091, "Filipino": 0.3806818181818182, "Spanish": 0.48863636363636365, "Chinese": 0.25, "Malay": 0.32954545454545453, "Vietnamese": 0.30113636363636365 }, "consistency_score_2": 0.3898809523809524, "consistency_score_3": 0.1876623376623377, "consistency_score_4": 0.10048701298701297, "consistency_score_5": 0.056006493506493504, "consistency_score_6": 0.02922077922077922, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.42613636363636365, "Indonesian,Filipino": 0.4431818181818182, "Indonesian,Spanish": 0.5113636363636364, "Indonesian,Chinese": 0.26136363636363635, "Indonesian,Malay": 0.5909090909090909, "Indonesian,Vietnamese": 0.4090909090909091, "English,Filipino": 0.36363636363636365, "English,Spanish": 0.6477272727272727, "English,Chinese": 0.2727272727272727, "English,Malay": 0.42045454545454547, "English,Vietnamese": 0.2840909090909091, "Filipino,Spanish": 0.42045454545454547, "Filipino,Chinese": 0.24431818181818182, "Filipino,Malay": 0.4659090909090909, "Filipino,Vietnamese": 0.48863636363636365, "Spanish,Chinese": 0.2556818181818182, "Spanish,Malay": 0.39204545454545453, "Spanish,Vietnamese": 0.4090909090909091, "Chinese,Malay": 0.25, "Chinese,Vietnamese": 0.20454545454545456, "Malay,Vietnamese": 0.42613636363636365 }, "3_combine": { "Indonesian,English,Filipino": 0.21022727272727273, "Indonesian,English,Spanish": 0.3465909090909091, "Indonesian,English,Chinese": 0.10795454545454546, "Indonesian,English,Malay": 0.2897727272727273, "Indonesian,English,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish": 0.24431818181818182, "Indonesian,Filipino,Chinese": 0.11931818181818182, "Indonesian,Filipino,Malay": 0.30113636363636365, "Indonesian,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Chinese": 0.13068181818181818, "Indonesian,Spanish,Malay": 0.29545454545454547, "Indonesian,Spanish,Vietnamese": 0.25, "Indonesian,Chinese,Malay": 0.16477272727272727, "Indonesian,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish": 0.2784090909090909, "English,Filipino,Chinese": 0.08522727272727272, "English,Filipino,Malay": 0.2215909090909091, "English,Filipino,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese": 0.17045454545454544, "English,Spanish,Malay": 0.29545454545454547, "English,Spanish,Vietnamese": 0.2556818181818182, "English,Chinese,Malay": 0.09659090909090909, "English,Chinese,Vietnamese": 0.056818181818181816, "English,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese": 0.10795454545454546, "Filipino,Spanish,Malay": 0.22727272727272727, "Filipino,Spanish,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay": 0.125, "Filipino,Chinese,Vietnamese": 0.11931818181818182, "Filipino,Malay,Vietnamese": 0.25, "Spanish,Chinese,Malay": 0.09659090909090909, "Spanish,Chinese,Vietnamese": 0.09090909090909091, "Spanish,Malay,Vietnamese": 0.21022727272727273, "Chinese,Malay,Vietnamese": 0.11363636363636363 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.17613636363636365, "Indonesian,English,Filipino,Chinese": 0.045454545454545456, "Indonesian,English,Filipino,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Chinese": 0.09090909090909091, "Indonesian,English,Spanish,Malay": 0.23295454545454544, "Indonesian,English,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,English,Chinese,Malay": 0.07954545454545454, "Indonesian,English,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,English,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Chinese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, "Indonesian,Filipino,Chinese,Vietnamese": 0.0625, "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay": 0.07954545454545454, "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Chinese": 0.07386363636363637, "English,Filipino,Spanish,Malay": 0.16477272727272727, "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, "English,Filipino,Chinese,Malay": 0.05113636363636364, "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Malay,Vietnamese": 0.10227272727272728, "English,Spanish,Chinese,Malay": 0.07954545454545454, "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, "English,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Chinese,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.03977272727272727, "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.03409090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, "AC3_2": 0.3889316769686338, "AC3_3": 0.2529684077467979, "AC3_4": 0.159630416078295, "AC3_5": 0.09788337802586573, "AC3_6": 0.05434837534702182, "AC3_7": 0.02208056170917894 }, "prompt_5": { "overall_acc": 0.3928571428571429, "language_acc": { "Indonesian": 0.3693181818181818, "English": 0.6022727272727273, "Filipino": 0.36363636363636365, "Spanish": 0.5056818181818182, "Chinese": 0.2727272727272727, "Malay": 0.3352272727272727, "Vietnamese": 0.30113636363636365 }, "consistency_score_2": 0.38988095238095244, "consistency_score_3": 0.18879870129870135, "consistency_score_4": 0.10162337662337663, "consistency_score_5": 0.056547619047619034, "consistency_score_6": 0.029220779220779224, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.42045454545454547, "Indonesian,Filipino": 0.4659090909090909, "Indonesian,Spanish": 0.48295454545454547, "Indonesian,Chinese": 0.23863636363636365, "Indonesian,Malay": 0.5909090909090909, "Indonesian,Vietnamese": 0.42613636363636365, "English,Filipino": 0.3465909090909091, "English,Spanish": 0.6647727272727273, "English,Chinese": 0.24431818181818182, "English,Malay": 0.42045454545454547, "English,Vietnamese": 0.30113636363636365, "Filipino,Spanish": 0.42045454545454547, "Filipino,Chinese": 0.25, "Filipino,Malay": 0.4602272727272727, "Filipino,Vietnamese": 0.48295454545454547, "Spanish,Chinese": 0.23295454545454544, "Spanish,Malay": 0.42045454545454547, "Spanish,Vietnamese": 0.3977272727272727, "Chinese,Malay": 0.26136363636363635, "Chinese,Vietnamese": 0.23295454545454544, "Malay,Vietnamese": 0.42613636363636365 }, "3_combine": { "Indonesian,English,Filipino": 0.20454545454545456, "Indonesian,English,Spanish": 0.3352272727272727, "Indonesian,English,Chinese": 0.09090909090909091, "Indonesian,English,Malay": 0.2840909090909091, "Indonesian,English,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish": 0.24431818181818182, "Indonesian,Filipino,Chinese": 0.125, "Indonesian,Filipino,Malay": 0.30113636363636365, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.11363636363636363, "Indonesian,Spanish,Malay": 0.29545454545454547, "Indonesian,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Chinese,Malay": 0.16477272727272727, "Indonesian,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish": 0.2727272727272727, "English,Filipino,Chinese": 0.06818181818181818, "English,Filipino,Malay": 0.2159090909090909, "English,Filipino,Vietnamese": 0.1534090909090909, "English,Spanish,Chinese": 0.1534090909090909, "English,Spanish,Malay": 0.3125, "English,Spanish,Vietnamese": 0.2556818181818182, "English,Chinese,Malay": 0.10795454545454546, "English,Chinese,Vietnamese": 0.07386363636363637, "English,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese": 0.09659090909090909, "Filipino,Spanish,Malay": 0.23295454545454544, "Filipino,Spanish,Vietnamese": 0.23295454545454544, "Filipino,Chinese,Malay": 0.13636363636363635, "Filipino,Chinese,Vietnamese": 0.11931818181818182, "Filipino,Malay,Vietnamese": 0.2556818181818182, "Spanish,Chinese,Malay": 0.10795454545454546, "Spanish,Chinese,Vietnamese": 0.08522727272727272, "Spanish,Malay,Vietnamese": 0.2215909090909091, "Chinese,Malay,Vietnamese": 0.125 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.16477272727272727, "Indonesian,English,Filipino,Chinese": 0.03409090909090909, "Indonesian,English,Filipino,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Chinese": 0.07386363636363637, "Indonesian,English,Spanish,Malay": 0.22727272727272727, "Indonesian,English,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,English,Chinese,Malay": 0.07386363636363637, "Indonesian,English,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.0625, "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, "Indonesian,Filipino,Chinese,Vietnamese": 0.06818181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Chinese": 0.056818181818181816, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay": 0.05113636363636364, "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.07954545454545454, "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, "English,Spanish,Malay,Vietnamese": 0.14772727272727273, "English,Chinese,Malay,Vietnamese": 0.056818181818181816, "Filipino,Spanish,Chinese,Malay": 0.0625, "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, "Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.11931818181818182, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay": 0.0625, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, "AC3_2": 0.3913633894122495, "AC3_3": 0.2550336907978269, "AC3_4": 0.16147640930980742, "AC3_5": 0.09886471142549386, "AC3_6": 0.05439560438271683, "AC3_7": 0.02208835340819019 } }, "sg_eval": { "prompt_1": { "accuracy": 0.5631067961165048 }, "prompt_2": { "accuracy": 0.5533980582524272 }, "prompt_3": { "accuracy": 0.5533980582524272 }, "prompt_4": { "accuracy": 0.5728155339805825 }, "prompt_5": { "accuracy": 0.5631067961165048 } }, "cn_eval": { "prompt_1": { "accuracy": 0.18095238095238095 }, "prompt_2": { "accuracy": 0.18095238095238095 }, "prompt_3": { "accuracy": 0.20952380952380953 }, "prompt_4": { "accuracy": 0.17142857142857143 }, "prompt_5": { "accuracy": 0.20952380952380953 } }, "us_eval": { "prompt_1": { "accuracy": 0.6542056074766355 }, "prompt_2": { "accuracy": 0.6355140186915887 }, "prompt_3": { "accuracy": 0.6728971962616822 }, "prompt_4": { "accuracy": 0.6822429906542056 }, "prompt_5": { "accuracy": 0.6728971962616822 } }, "ph_eval": { "prompt_1": { "accuracy": 0.32, "category_acc": { "brand": 0.2, "demographics": 0.0, "biology": 0.4, "history": 0.13333333333333333, "literature": 0.3, "politics": 0.7, "culture": 0.4, "film": 0.3, "law": 0.2, "geography": 0.5 } }, "prompt_2": { "accuracy": 0.29, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.4, "history": 0.06666666666666667, "literature": 0.3, "politics": 0.5, "culture": 0.4, "film": 0.3, "law": 0.2, "geography": 0.4 } }, "prompt_3": { "accuracy": 0.34, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.4, "history": 0.13333333333333333, "literature": 0.3, "politics": 0.7, "culture": 0.4, "film": 0.3, "law": 0.2, "geography": 0.5 } }, "prompt_4": { "accuracy": 0.35, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.2, "literature": 0.3, "politics": 0.7, "culture": 0.4, "film": 0.3, "law": 0.2, "geography": 0.5 } }, "prompt_5": { "accuracy": 0.35, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.5, "history": 0.13333333333333333, "literature": 0.4, "politics": 0.6, "culture": 0.4, "film": 0.3, "law": 0.2, "geography": 0.5 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.17143237165650724 }, "prompt_2": { "bleu_score": 0.17307422379138457 }, "prompt_3": { "bleu_score": 0.17719935633819425 }, "prompt_4": { "bleu_score": 0.17445808927858594 }, "prompt_5": { "bleu_score": 0.17398886376784886 } }, "indommlu": { "prompt_1": { "accuracy": 0.3235863542292543, "category_acc": { "History": 0.29116465863453816, "Geography": 0.25918367346938775, "Lampungic": 0.2585034013605442, "Social science": 0.41736227045075125, "Balinese": 0.2951167728237792, "Makassarese": 0.3010752688172043, "Banjarese": 0.2569444444444444, "Chemistry": 0.21897810218978103, "Biology": 0.31005917159763313, "Science": 0.3302373581011352, "Christian religion": 0.3781094527363184, "Art": 0.3544093178036606, "Islam religion": 0.3456614509246088, "Hindu religion": 0.3, "Madurese": 0.31186440677966104, "Sport": 0.3716216216216216, "Indonesian language": 0.37484433374844334, "Physics": 0.24646464646464647, "Minangkabau culture": 0.271356783919598, "Dayak language": 0.25688073394495414, "Sociology": 0.3165322580645161, "Economy": 0.3073770491803279, "Sundanese": 0.31892826274848746, "Javanese": 0.2772177419354839, "Civic education": 0.34334763948497854 } }, "prompt_2": { "accuracy": 0.3191134254623139, "category_acc": { "History": 0.3112449799196787, "Geography": 0.24897959183673468, "Lampungic": 0.3401360544217687, "Social science": 0.3989983305509182, "Balinese": 0.28874734607218683, "Makassarese": 0.3279569892473118, "Banjarese": 0.2847222222222222, "Chemistry": 0.23065693430656933, "Biology": 0.3005917159763314, "Science": 0.31682146542827655, "Christian religion": 0.38308457711442784, "Art": 0.3211314475873544, "Islam religion": 0.31152204836415365, "Hindu religion": 0.29333333333333333, "Madurese": 0.2983050847457627, "Sport": 0.39864864864864863, "Indonesian language": 0.3664383561643836, "Physics": 0.2545454545454545, "Minangkabau culture": 0.27638190954773867, "Dayak language": 0.23853211009174313, "Sociology": 0.3225806451612903, "Economy": 0.29713114754098363, "Sundanese": 0.324114088159032, "Javanese": 0.2842741935483871, "Civic education": 0.33047210300429186 } }, "prompt_3": { "accuracy": 0.31857934441551505, "category_acc": { "History": 0.2971887550200803, "Geography": 0.24693877551020407, "Lampungic": 0.2857142857142857, "Social science": 0.41402337228714525, "Balinese": 0.2802547770700637, "Makassarese": 0.34946236559139787, "Banjarese": 0.2708333333333333, "Chemistry": 0.21897810218978103, "Biology": 0.2982248520710059, "Science": 0.32920536635706915, "Christian religion": 0.373134328358209, "Art": 0.3261231281198003, "Islam religion": 0.33712660028449504, "Hindu religion": 0.28, "Madurese": 0.2677966101694915, "Sport": 0.39864864864864863, "Indonesian language": 0.3707970112079701, "Physics": 0.24848484848484848, "Minangkabau culture": 0.25125628140703515, "Dayak language": 0.26605504587155965, "Sociology": 0.31451612903225806, "Economy": 0.2766393442622951, "Sundanese": 0.3292999135695765, "Javanese": 0.27419354838709675, "Civic education": 0.33047210300429186 } }, "prompt_4": { "accuracy": 0.30616196007744173, "category_acc": { "History": 0.2991967871485944, "Geography": 0.27346938775510204, "Lampungic": 0.3197278911564626, "Social science": 0.3672787979966611, "Balinese": 0.3184713375796178, "Makassarese": 0.3924731182795699, "Banjarese": 0.25, "Chemistry": 0.22481751824817517, "Biology": 0.293491124260355, "Science": 0.2837977296181631, "Christian religion": 0.3283582089552239, "Art": 0.31281198003327787, "Islam religion": 0.30867709815078237, "Hindu religion": 0.3, "Madurese": 0.28135593220338984, "Sport": 0.2972972972972973, "Indonesian language": 0.35211706102117063, "Physics": 0.24646464646464647, "Minangkabau culture": 0.2814070351758794, "Dayak language": 0.30275229357798167, "Sociology": 0.30443548387096775, "Economy": 0.23975409836065573, "Sundanese": 0.317199654278306, "Javanese": 0.2560483870967742, "Civic education": 0.32331902718168815 } }, "prompt_5": { "accuracy": 0.32592295880899924, "category_acc": { "History": 0.2991967871485944, "Geography": 0.2836734693877551, "Lampungic": 0.29931972789115646, "Social science": 0.4307178631051753, "Balinese": 0.28662420382165604, "Makassarese": 0.3333333333333333, "Banjarese": 0.24305555555555555, "Chemistry": 0.2510948905109489, "Biology": 0.2982248520710059, "Science": 0.32301341589267285, "Christian religion": 0.36318407960199006, "Art": 0.34608985024958405, "Islam religion": 0.3357041251778094, "Hindu religion": 0.30666666666666664, "Madurese": 0.30847457627118646, "Sport": 0.3918918918918919, "Indonesian language": 0.37359900373599003, "Physics": 0.25252525252525254, "Minangkabau culture": 0.2814070351758794, "Dayak language": 0.28440366972477066, "Sociology": 0.3084677419354839, "Economy": 0.28688524590163933, "Sundanese": 0.3327571305099395, "Javanese": 0.2752016129032258, "Civic education": 0.3547925608011445 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.28907320267771625 }, "prompt_2": { "bleu_score": 0.2893892328430556 }, "prompt_3": { "bleu_score": 0.2886733513466506 }, "prompt_4": { "bleu_score": 0.29284370210142285 }, "prompt_5": { "bleu_score": 0.2747670406496726 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.07158636430680222 }, "prompt_2": { "bleu_score": 0.07103723230064819 }, "prompt_3": { "bleu_score": 0.07070585299769382 }, "prompt_4": { "bleu_score": 0.07161514805550445 }, "prompt_5": { "bleu_score": 0.06633933982982537 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.018213298322017683 }, "prompt_2": { "bleu_score": 0.01861580300222175 }, "prompt_3": { "bleu_score": 0.01940857812030854 }, "prompt_4": { "bleu_score": 0.015585331792968693 }, "prompt_5": { "bleu_score": 0.018660857793531816 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.27139437453475074 }, "prompt_2": { "bleu_score": 0.26884915209700333 }, "prompt_3": { "bleu_score": 0.26967135172638884 }, "prompt_4": { "bleu_score": 0.2718225078074264 }, "prompt_5": { "bleu_score": 0.25925138729269165 } }, "mmlu": { "prompt_1": { "accuracy": 0.5530921820303384 }, "prompt_2": { "accuracy": 0.558926487747958 }, "prompt_3": { "accuracy": 0.544924154025671 }, "prompt_4": { "accuracy": 0.5472578763127188 }, "prompt_5": { "accuracy": 0.5577596266044341 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.5392205934930282, "category_acc": { "high_school_european_history": 0.7378048780487805, "business_ethics": 0.7070707070707071, "clinical_knowledge": 0.6060606060606061, "medical_genetics": 0.5252525252525253, "high_school_us_history": 0.7684729064039408, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.7711864406779662, "virology": 0.4909090909090909, "high_school_microeconomics": 0.5864978902953587, "econometrics": 0.35398230088495575, "college_computer_science": 0.4444444444444444, "high_school_biology": 0.6634304207119741, "abstract_algebra": 0.20202020202020202, "professional_accounting": 0.4306049822064057, "philosophy": 0.5580645161290323, "professional_medicine": 0.5571955719557196, "nutrition": 0.6065573770491803, "global_facts": 0.2828282828282828, "machine_learning": 0.40540540540540543, "security_studies": 0.6557377049180327, "public_relations": 0.5504587155963303, "professional_psychology": 0.5515548281505729, "prehistory": 0.5913312693498453, "anatomy": 0.47761194029850745, "human_sexuality": 0.6538461538461539, "college_medicine": 0.48255813953488375, "high_school_government_and_politics": 0.8333333333333334, "college_chemistry": 0.3838383838383838, "logical_fallacies": 0.6666666666666666, "high_school_geography": 0.7258883248730964, "elementary_mathematics": 0.3740053050397878, "human_aging": 0.6036036036036037, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.7794117647058824, "formal_logic": 0.288, "high_school_statistics": 0.4, "international_law": 0.7, "high_school_mathematics": 0.2379182156133829, "high_school_computer_science": 0.4444444444444444, "conceptual_physics": 0.452991452991453, "miscellaneous": 0.7186700767263428, "high_school_chemistry": 0.40594059405940597, "marketing": 0.8369098712446352, "professional_law": 0.4253098499673842, "management": 0.7941176470588235, "college_physics": 0.42574257425742573, "jurisprudence": 0.616822429906542, "world_religions": 0.7, "sociology": 0.68, "us_foreign_policy": 0.7474747474747475, "high_school_macroeconomics": 0.5295629820051414, "computer_security": 0.6363636363636364, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6405797101449275, "electrical_engineering": 0.5277777777777778, "astronomy": 0.5629139072847682, "college_biology": 0.5104895104895105 } }, "prompt_2": { "accuracy": 0.5386485520200215, "category_acc": { "high_school_european_history": 0.7439024390243902, "business_ethics": 0.7272727272727273, "clinical_knowledge": 0.6022727272727273, "medical_genetics": 0.5252525252525253, "high_school_us_history": 0.7684729064039408, "high_school_physics": 0.26, "high_school_world_history": 0.7627118644067796, "virology": 0.47878787878787876, "high_school_microeconomics": 0.5738396624472574, "econometrics": 0.34513274336283184, "college_computer_science": 0.45454545454545453, "high_school_biology": 0.6601941747572816, "abstract_algebra": 0.1919191919191919, "professional_accounting": 0.4234875444839858, "philosophy": 0.5709677419354838, "professional_medicine": 0.5682656826568265, "nutrition": 0.6032786885245902, "global_facts": 0.2727272727272727, "machine_learning": 0.40540540540540543, "security_studies": 0.6639344262295082, "public_relations": 0.5412844036697247, "professional_psychology": 0.5482815057283142, "prehistory": 0.5913312693498453, "anatomy": 0.4925373134328358, "human_sexuality": 0.6384615384615384, "college_medicine": 0.48255813953488375, "high_school_government_and_politics": 0.8333333333333334, "college_chemistry": 0.3838383838383838, "logical_fallacies": 0.6604938271604939, "high_school_geography": 0.7055837563451777, "elementary_mathematics": 0.3687002652519894, "human_aging": 0.5900900900900901, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.7738970588235294, "formal_logic": 0.312, "high_school_statistics": 0.386046511627907, "international_law": 0.7, "high_school_mathematics": 0.24907063197026022, "high_school_computer_science": 0.43434343434343436, "conceptual_physics": 0.452991452991453, "miscellaneous": 0.7199488491048593, "high_school_chemistry": 0.39603960396039606, "marketing": 0.8326180257510729, "professional_law": 0.4298760600130463, "management": 0.803921568627451, "college_physics": 0.40594059405940597, "jurisprudence": 0.6448598130841121, "world_religions": 0.6882352941176471, "sociology": 0.69, "us_foreign_policy": 0.7474747474747475, "high_school_macroeconomics": 0.5398457583547558, "computer_security": 0.6464646464646465, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6521739130434783, "electrical_engineering": 0.4861111111111111, "astronomy": 0.5695364238410596, "college_biology": 0.5104895104895105 } }, "prompt_3": { "accuracy": 0.5380765105470147, "category_acc": { "high_school_european_history": 0.75, "business_ethics": 0.7070707070707071, "clinical_knowledge": 0.6022727272727273, "medical_genetics": 0.5151515151515151, "high_school_us_history": 0.7733990147783252, "high_school_physics": 0.28, "high_school_world_history": 0.7669491525423728, "virology": 0.48484848484848486, "high_school_microeconomics": 0.569620253164557, "econometrics": 0.34513274336283184, "college_computer_science": 0.4444444444444444, "high_school_biology": 0.6731391585760518, "abstract_algebra": 0.21212121212121213, "professional_accounting": 0.4234875444839858, "philosophy": 0.5741935483870968, "professional_medicine": 0.5682656826568265, "nutrition": 0.6131147540983607, "global_facts": 0.2727272727272727, "machine_learning": 0.3783783783783784, "security_studies": 0.6557377049180327, "public_relations": 0.5779816513761468, "professional_psychology": 0.5450081833060556, "prehistory": 0.5789473684210527, "anatomy": 0.4701492537313433, "human_sexuality": 0.6384615384615384, "college_medicine": 0.47093023255813954, "high_school_government_and_politics": 0.84375, "college_chemistry": 0.3939393939393939, "logical_fallacies": 0.6604938271604939, "high_school_geography": 0.7208121827411168, "elementary_mathematics": 0.36074270557029176, "human_aging": 0.5765765765765766, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.78125, "formal_logic": 0.328, "high_school_statistics": 0.386046511627907, "international_law": 0.7, "high_school_mathematics": 0.2379182156133829, "high_school_computer_science": 0.4444444444444444, "conceptual_physics": 0.43162393162393164, "miscellaneous": 0.7071611253196931, "high_school_chemistry": 0.40594059405940597, "marketing": 0.8326180257510729, "professional_law": 0.43118069145466403, "management": 0.7745098039215687, "college_physics": 0.42574257425742573, "jurisprudence": 0.6261682242990654, "world_religions": 0.6882352941176471, "sociology": 0.69, "us_foreign_policy": 0.7575757575757576, "high_school_macroeconomics": 0.5372750642673522, "computer_security": 0.6363636363636364, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6521739130434783, "electrical_engineering": 0.5069444444444444, "astronomy": 0.5827814569536424, "college_biology": 0.4965034965034965 } }, "prompt_4": { "accuracy": 0.5392920986771541, "category_acc": { "high_school_european_history": 0.75, "business_ethics": 0.7272727272727273, "clinical_knowledge": 0.6060606060606061, "medical_genetics": 0.5151515151515151, "high_school_us_history": 0.7684729064039408, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.7627118644067796, "virology": 0.48484848484848486, "high_school_microeconomics": 0.5780590717299579, "econometrics": 0.336283185840708, "college_computer_science": 0.46464646464646464, "high_school_biology": 0.6666666666666666, "abstract_algebra": 0.18181818181818182, "professional_accounting": 0.4377224199288256, "philosophy": 0.5741935483870968, "professional_medicine": 0.5535055350553506, "nutrition": 0.6131147540983607, "global_facts": 0.2727272727272727, "machine_learning": 0.38738738738738737, "security_studies": 0.6639344262295082, "public_relations": 0.5504587155963303, "professional_psychology": 0.5499181669394435, "prehistory": 0.5913312693498453, "anatomy": 0.4701492537313433, "human_sexuality": 0.6384615384615384, "college_medicine": 0.47674418604651164, "high_school_government_and_politics": 0.828125, "college_chemistry": 0.3939393939393939, "logical_fallacies": 0.6728395061728395, "high_school_geography": 0.7309644670050761, "elementary_mathematics": 0.3687002652519894, "human_aging": 0.5630630630630631, "college_mathematics": 0.32323232323232326, "high_school_psychology": 0.78125, "formal_logic": 0.32, "high_school_statistics": 0.4046511627906977, "international_law": 0.7166666666666667, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.4444444444444444, "conceptual_physics": 0.4230769230769231, "miscellaneous": 0.7071611253196931, "high_school_chemistry": 0.4158415841584158, "marketing": 0.8240343347639485, "professional_law": 0.4292237442922374, "management": 0.7843137254901961, "college_physics": 0.39603960396039606, "jurisprudence": 0.6448598130841121, "world_religions": 0.711764705882353, "sociology": 0.695, "us_foreign_policy": 0.7373737373737373, "high_school_macroeconomics": 0.5449871465295629, "computer_security": 0.6464646464646465, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6405797101449275, "electrical_engineering": 0.5138888888888888, "astronomy": 0.5761589403973509, "college_biology": 0.5034965034965035 } }, "prompt_5": { "accuracy": 0.5432248838040757, "category_acc": { "high_school_european_history": 0.75, "business_ethics": 0.7373737373737373, "clinical_knowledge": 0.6022727272727273, "medical_genetics": 0.5252525252525253, "high_school_us_history": 0.7832512315270936, "high_school_physics": 0.26, "high_school_world_history": 0.7627118644067796, "virology": 0.4666666666666667, "high_school_microeconomics": 0.5738396624472574, "econometrics": 0.36283185840707965, "college_computer_science": 0.45454545454545453, "high_school_biology": 0.6634304207119741, "abstract_algebra": 0.1919191919191919, "professional_accounting": 0.4234875444839858, "philosophy": 0.5709677419354838, "professional_medicine": 0.5535055350553506, "nutrition": 0.6295081967213115, "global_facts": 0.31313131313131315, "machine_learning": 0.38738738738738737, "security_studies": 0.6762295081967213, "public_relations": 0.5321100917431193, "professional_psychology": 0.55810147299509, "prehistory": 0.6068111455108359, "anatomy": 0.47761194029850745, "human_sexuality": 0.6538461538461539, "college_medicine": 0.47674418604651164, "high_school_government_and_politics": 0.8333333333333334, "college_chemistry": 0.3838383838383838, "logical_fallacies": 0.6728395061728395, "high_school_geography": 0.7360406091370558, "elementary_mathematics": 0.3740053050397878, "human_aging": 0.5900900900900901, "college_mathematics": 0.32323232323232326, "high_school_psychology": 0.7886029411764706, "formal_logic": 0.32, "high_school_statistics": 0.4, "international_law": 0.7083333333333334, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.46464646464646464, "conceptual_physics": 0.4230769230769231, "miscellaneous": 0.7212276214833759, "high_school_chemistry": 0.40594059405940597, "marketing": 0.8283261802575107, "professional_law": 0.43313763861709065, "management": 0.7941176470588235, "college_physics": 0.40594059405940597, "jurisprudence": 0.6822429906542056, "world_religions": 0.7235294117647059, "sociology": 0.69, "us_foreign_policy": 0.7575757575757576, "high_school_macroeconomics": 0.5295629820051414, "computer_security": 0.6666666666666666, "moral_scenarios": 0.23825503355704697, "moral_disputes": 0.6608695652173913, "electrical_engineering": 0.5, "astronomy": 0.5761589403973509, "college_biology": 0.48951048951048953 } } }, "c_eval": { "prompt_1": { "accuracy": 0.2473997028231798 }, "prompt_2": { "accuracy": 0.24294205052005943 }, "prompt_3": { "accuracy": 0.2526002971768202 }, "prompt_4": { "accuracy": 0.24814264487369986 }, "prompt_5": { "accuracy": 0.24962852897473997 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.2559153175591532, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.20833333333333334, "computer_architecture": 0.19230769230769232, "college_programming": 0.3333333333333333, "college_physics": 0.20833333333333334, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.13793103448275862, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.25, "middle_school_biology": 0.2692307692307692, "middle_school_physics": 0.20833333333333334, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.3, "business_administration": 0.18421052631578946, "marxism": 0.25, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.23529411764705882, "teacher_qualification": 0.2857142857142857, "high_school_politics": 0.3333333333333333, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.058823529411764705, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.4074074074074074, "law": 0.20689655172413793, "chinese_language_and_literature": 0.25, "art_studies": 0.23684210526315788, "professional_tour_guide": 0.3235294117647059, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.28, "middle_school_history": 0.18518518518518517, "civil_servant": 0.19230769230769232, "sports_science": 0.125, "plant_protection": 0.14814814814814814, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.35294117647058826, "accountant": 0.2777777777777778, "fire_engineer": 0.1388888888888889, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.2037037037037037, "physician": 0.25925925925925924 } }, "prompt_2": { "accuracy": 0.26089663760896636, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.16666666666666666, "computer_architecture": 0.19230769230769232, "college_programming": 0.2857142857142857, "college_physics": 0.20833333333333334, "college_chemistry": 0.41379310344827586, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.47619047619047616, "electrical_engineer": 0.21428571428571427, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.16666666666666666, "high_school_biology": 0.25, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.2692307692307692, "middle_school_physics": 0.20833333333333334, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.26666666666666666, "business_administration": 0.21052631578947367, "marxism": 0.25, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.17647058823529413, "teacher_qualification": 0.30612244897959184, "high_school_politics": 0.3333333333333333, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.058823529411764705, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.4074074074074074, "law": 0.20689655172413793, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.2631578947368421, "professional_tour_guide": 0.3235294117647059, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.28, "middle_school_history": 0.18518518518518517, "civil_servant": 0.19230769230769232, "sports_science": 0.20833333333333334, "plant_protection": 0.14814814814814814, "basic_medicine": 0.16666666666666666, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.37254901960784315, "accountant": 0.2962962962962963, "fire_engineer": 0.1111111111111111, "environmental_impact_assessment_engineer": 0.16666666666666666, "tax_accountant": 0.24074074074074073, "physician": 0.2962962962962963 } }, "prompt_3": { "accuracy": 0.26089663760896636, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.16666666666666666, "computer_architecture": 0.19230769230769232, "college_programming": 0.2619047619047619, "college_physics": 0.25, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.13793103448275862, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.25, "middle_school_mathematics": 0.25, "middle_school_biology": 0.3076923076923077, "middle_school_physics": 0.25, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.26666666666666666, "business_administration": 0.18421052631578946, "marxism": 0.25, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.2647058823529412, "teacher_qualification": 0.2653061224489796, "high_school_politics": 0.3333333333333333, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.058823529411764705, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.4074074074074074, "law": 0.20689655172413793, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.2894736842105263, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.32, "middle_school_history": 0.18518518518518517, "civil_servant": 0.21153846153846154, "sports_science": 0.16666666666666666, "plant_protection": 0.14814814814814814, "basic_medicine": 0.25, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.37254901960784315, "accountant": 0.2777777777777778, "fire_engineer": 0.1388888888888889, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.2222222222222222, "physician": 0.24074074074074073 } }, "prompt_4": { "accuracy": 0.2671232876712329, "category_acc": { "computer_network": 0.20833333333333334, "operating_system": 0.16666666666666666, "computer_architecture": 0.19230769230769232, "college_programming": 0.2619047619047619, "college_physics": 0.2916666666666667, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.391304347826087, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.25, "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.125, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.25, "business_administration": 0.21052631578947367, "marxism": 0.3333333333333333, "mao_zedong_thought": 0.27586206896551724, "education_science": 0.23529411764705882, "teacher_qualification": 0.2857142857142857, "high_school_politics": 0.3333333333333333, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.37037037037037035, "law": 0.3448275862068966, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.38235294117647056, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.36, "middle_school_history": 0.2222222222222222, "civil_servant": 0.2692307692307692, "sports_science": 0.16666666666666666, "plant_protection": 0.2222222222222222, "basic_medicine": 0.16666666666666666, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.2777777777777778, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.25925925925925924, "physician": 0.24074074074074073 } }, "prompt_5": { "accuracy": 0.2577833125778331, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.16666666666666666, "computer_architecture": 0.11538461538461539, "college_programming": 0.2619047619047619, "college_physics": 0.20833333333333334, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.42857142857142855, "electrical_engineer": 0.21428571428571427, "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.2692307692307692, "middle_school_physics": 0.20833333333333334, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.3333333333333333, "business_administration": 0.18421052631578946, "marxism": 0.25, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.2647058823529412, "teacher_qualification": 0.2857142857142857, "high_school_politics": 0.3333333333333333, "high_school_geography": 0.16666666666666666, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.058823529411764705, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.375, "logic": 0.4074074074074074, "law": 0.20689655172413793, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.2631578947368421, "professional_tour_guide": 0.3235294117647059, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.32, "middle_school_history": 0.18518518518518517, "civil_servant": 0.21153846153846154, "sports_science": 0.16666666666666666, "plant_protection": 0.14814814814814814, "basic_medicine": 0.2916666666666667, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.37254901960784315, "accountant": 0.2777777777777778, "fire_engineer": 0.1111111111111111, "environmental_impact_assessment_engineer": 0.16666666666666666, "tax_accountant": 0.2037037037037037, "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.2724014336917563 }, "prompt_2": { "accuracy": 0.30824372759856633 }, "prompt_3": { "accuracy": 0.27956989247311825 }, "prompt_4": { "accuracy": 0.2939068100358423 }, "prompt_5": { "accuracy": 0.2831541218637993 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2518563287860473, "category_acc": { "agronomy": 0.23076923076923078, "anatomy": 0.2905405405405405, "ancient_chinese": 0.25609756097560976, "arts": 0.2375, "astronomy": 0.21818181818181817, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.3125, "chinese_driving_rule": 0.32061068702290074, "chinese_food_culture": 0.27941176470588236, "chinese_foreign_policy": 0.2897196261682243, "chinese_history": 0.2631578947368421, "chinese_literature": 0.27450980392156865, "chinese_teacher_qualification": 0.2569832402234637, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.25471698113207547, "college_education": 0.29906542056074764, "college_engineering_hydrology": 0.2830188679245283, "college_law": 0.23148148148148148, "college_mathematics": 0.17142857142857143, "college_medical_statistics": 0.16037735849056603, "college_medicine": 0.23809523809523808, "computer_science": 0.29901960784313725, "computer_security": 0.2807017543859649, "conceptual_physics": 0.25170068027210885, "construction_project_management": 0.2302158273381295, "economics": 0.24528301886792453, "education": 0.1901840490797546, "electrical_engineering": 0.20348837209302326, "elementary_chinese": 0.25396825396825395, "elementary_commonsense": 0.18686868686868688, "elementary_information_and_technology": 0.29831932773109243, "elementary_mathematics": 0.26521739130434785, "ethnology": 0.2740740740740741, "food_science": 0.2937062937062937, "genetics": 0.2784090909090909, "global_facts": 0.28187919463087246, "high_school_biology": 0.23668639053254437, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.2288135593220339, "high_school_mathematics": 0.23170731707317074, "high_school_physics": 0.22727272727272727, "high_school_politics": 0.26573426573426573, "human_sexuality": 0.25396825396825395, "international_law": 0.2864864864864865, "journalism": 0.2558139534883721, "jurisprudence": 0.22871046228710462, "legal_and_moral_basis": 0.26635514018691586, "logical": 0.2682926829268293, "machine_learning": 0.2540983606557377, "management": 0.2523809523809524, "marketing": 0.2388888888888889, "marxist_theory": 0.24338624338624337, "modern_chinese": 0.27586206896551724, "nutrition": 0.30344827586206896, "philosophy": 0.21904761904761905, "professional_accounting": 0.21142857142857144, "professional_law": 0.2890995260663507, "professional_medicine": 0.2712765957446808, "professional_psychology": 0.1896551724137931, "public_relations": 0.23563218390804597, "security_study": 0.26666666666666666, "sociology": 0.22566371681415928, "sports_science": 0.23030303030303031, "traditional_chinese_medicine": 0.23783783783783785, "virology": 0.2485207100591716, "world_history": 0.2546583850931677, "world_religions": 0.25625 } }, "prompt_2": { "accuracy": 0.25392850975651876, "category_acc": { "agronomy": 0.27218934911242604, "anatomy": 0.27702702702702703, "ancient_chinese": 0.27439024390243905, "arts": 0.29375, "astronomy": 0.2, "business_ethics": 0.22009569377990432, "chinese_civil_service_exam": 0.2875, "chinese_driving_rule": 0.3282442748091603, "chinese_food_culture": 0.33088235294117646, "chinese_foreign_policy": 0.24299065420560748, "chinese_history": 0.25696594427244585, "chinese_literature": 0.23529411764705882, "chinese_teacher_qualification": 0.24581005586592178, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.2830188679245283, "college_education": 0.2803738317757009, "college_engineering_hydrology": 0.3018867924528302, "college_law": 0.24074074074074073, "college_mathematics": 0.23809523809523808, "college_medical_statistics": 0.1792452830188679, "college_medicine": 0.23443223443223443, "computer_science": 0.30392156862745096, "computer_security": 0.3157894736842105, "conceptual_physics": 0.2653061224489796, "construction_project_management": 0.26618705035971224, "economics": 0.2389937106918239, "education": 0.2392638036809816, "electrical_engineering": 0.2558139534883721, "elementary_chinese": 0.23412698412698413, "elementary_commonsense": 0.19696969696969696, "elementary_information_and_technology": 0.27310924369747897, "elementary_mathematics": 0.26521739130434785, "ethnology": 0.28888888888888886, "food_science": 0.2727272727272727, "genetics": 0.25, "global_facts": 0.24161073825503357, "high_school_biology": 0.23668639053254437, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.211864406779661, "high_school_mathematics": 0.2073170731707317, "high_school_physics": 0.23636363636363636, "high_school_politics": 0.27972027972027974, "human_sexuality": 0.23809523809523808, "international_law": 0.2972972972972973, "journalism": 0.27906976744186046, "jurisprudence": 0.24817518248175183, "legal_and_moral_basis": 0.2757009345794392, "logical": 0.24390243902439024, "machine_learning": 0.28688524590163933, "management": 0.23333333333333334, "marketing": 0.2611111111111111, "marxist_theory": 0.291005291005291, "modern_chinese": 0.28448275862068967, "nutrition": 0.27586206896551724, "philosophy": 0.2571428571428571, "professional_accounting": 0.21714285714285714, "professional_law": 0.2559241706161137, "professional_medicine": 0.21010638297872342, "professional_psychology": 0.25862068965517243, "public_relations": 0.22988505747126436, "security_study": 0.24444444444444444, "sociology": 0.2610619469026549, "sports_science": 0.23030303030303031, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.20118343195266272, "world_history": 0.2608695652173913, "world_religions": 0.2625 } }, "prompt_3": { "accuracy": 0.2532377827663616, "category_acc": { "agronomy": 0.26627218934911245, "anatomy": 0.33783783783783783, "ancient_chinese": 0.2682926829268293, "arts": 0.26875, "astronomy": 0.23636363636363636, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.28125, "chinese_driving_rule": 0.32061068702290074, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.2523364485981308, "chinese_history": 0.2631578947368421, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.26256983240223464, "clinical_knowledge": 0.2616033755274262, "college_actuarial_science": 0.2641509433962264, "college_education": 0.3364485981308411, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.21296296296296297, "college_mathematics": 0.2, "college_medical_statistics": 0.1792452830188679, "college_medicine": 0.23076923076923078, "computer_science": 0.28431372549019607, "computer_security": 0.27485380116959063, "conceptual_physics": 0.2925170068027211, "construction_project_management": 0.2589928057553957, "economics": 0.2893081761006289, "education": 0.25153374233128833, "electrical_engineering": 0.22093023255813954, "elementary_chinese": 0.23412698412698413, "elementary_commonsense": 0.2474747474747475, "elementary_information_and_technology": 0.2605042016806723, "elementary_mathematics": 0.26521739130434785, "ethnology": 0.2962962962962963, "food_science": 0.25874125874125875, "genetics": 0.23295454545454544, "global_facts": 0.2348993288590604, "high_school_biology": 0.22485207100591717, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.2288135593220339, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.22727272727272727, "high_school_politics": 0.2937062937062937, "human_sexuality": 0.2619047619047619, "international_law": 0.23783783783783785, "journalism": 0.21511627906976744, "jurisprudence": 0.23114355231143552, "legal_and_moral_basis": 0.2383177570093458, "logical": 0.3333333333333333, "machine_learning": 0.26229508196721313, "management": 0.23809523809523808, "marketing": 0.25555555555555554, "marxist_theory": 0.24338624338624337, "modern_chinese": 0.27586206896551724, "nutrition": 0.30344827586206896, "philosophy": 0.2, "professional_accounting": 0.22857142857142856, "professional_law": 0.25118483412322273, "professional_medicine": 0.23404255319148937, "professional_psychology": 0.21551724137931033, "public_relations": 0.27011494252873564, "security_study": 0.2518518518518518, "sociology": 0.252212389380531, "sports_science": 0.23636363636363636, "traditional_chinese_medicine": 0.25405405405405407, "virology": 0.1952662721893491, "world_history": 0.2670807453416149, "world_religions": 0.2875 } }, "prompt_4": { "accuracy": 0.25392850975651876, "category_acc": { "agronomy": 0.25443786982248523, "anatomy": 0.25675675675675674, "ancient_chinese": 0.2682926829268293, "arts": 0.24375, "astronomy": 0.19393939393939394, "business_ethics": 0.2535885167464115, "chinese_civil_service_exam": 0.24375, "chinese_driving_rule": 0.3053435114503817, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.27102803738317754, "chinese_history": 0.25077399380804954, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.24022346368715083, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.27358490566037735, "college_education": 0.2897196261682243, "college_engineering_hydrology": 0.3018867924528302, "college_law": 0.2222222222222222, "college_mathematics": 0.20952380952380953, "college_medical_statistics": 0.24528301886792453, "college_medicine": 0.2490842490842491, "computer_science": 0.30392156862745096, "computer_security": 0.2573099415204678, "conceptual_physics": 0.2857142857142857, "construction_project_management": 0.26618705035971224, "economics": 0.2389937106918239, "education": 0.25153374233128833, "electrical_engineering": 0.2441860465116279, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.2474747474747475, "elementary_information_and_technology": 0.2857142857142857, "elementary_mathematics": 0.26956521739130435, "ethnology": 0.23703703703703705, "food_science": 0.3006993006993007, "genetics": 0.2159090909090909, "global_facts": 0.2751677852348993, "high_school_biology": 0.23076923076923078, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.2457627118644068, "high_school_mathematics": 0.1951219512195122, "high_school_physics": 0.2727272727272727, "high_school_politics": 0.26573426573426573, "human_sexuality": 0.24603174603174602, "international_law": 0.2648648648648649, "journalism": 0.2441860465116279, "jurisprudence": 0.26277372262773724, "legal_and_moral_basis": 0.2570093457943925, "logical": 0.25203252032520324, "machine_learning": 0.22950819672131148, "management": 0.24285714285714285, "marketing": 0.25555555555555554, "marxist_theory": 0.25925925925925924, "modern_chinese": 0.27586206896551724, "nutrition": 0.2, "philosophy": 0.2761904761904762, "professional_accounting": 0.2571428571428571, "professional_law": 0.23696682464454977, "professional_medicine": 0.24202127659574468, "professional_psychology": 0.24568965517241378, "public_relations": 0.2413793103448276, "security_study": 0.2740740740740741, "sociology": 0.2610619469026549, "sports_science": 0.23636363636363636, "traditional_chinese_medicine": 0.2594594594594595, "virology": 0.25443786982248523, "world_history": 0.2360248447204969, "world_religions": 0.275 } }, "prompt_5": { "accuracy": 0.254101191504058, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.2972972972972973, "ancient_chinese": 0.2621951219512195, "arts": 0.325, "astronomy": 0.23030303030303031, "business_ethics": 0.23923444976076555, "chinese_civil_service_exam": 0.30625, "chinese_driving_rule": 0.31297709923664124, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.2523364485981308, "chinese_history": 0.2631578947368421, "chinese_literature": 0.24509803921568626, "chinese_teacher_qualification": 0.25139664804469275, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.27358490566037735, "college_education": 0.2897196261682243, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.25, "college_mathematics": 0.2, "college_medical_statistics": 0.16981132075471697, "college_medicine": 0.23443223443223443, "computer_science": 0.31862745098039214, "computer_security": 0.2573099415204678, "conceptual_physics": 0.272108843537415, "construction_project_management": 0.2446043165467626, "economics": 0.25157232704402516, "education": 0.2392638036809816, "electrical_engineering": 0.22093023255813954, "elementary_chinese": 0.21825396825396826, "elementary_commonsense": 0.2222222222222222, "elementary_information_and_technology": 0.2773109243697479, "elementary_mathematics": 0.2826086956521739, "ethnology": 0.3037037037037037, "food_science": 0.2727272727272727, "genetics": 0.26136363636363635, "global_facts": 0.2684563758389262, "high_school_biology": 0.21893491124260356, "high_school_chemistry": 0.2196969696969697, "high_school_geography": 0.2796610169491525, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.21818181818181817, "high_school_politics": 0.2937062937062937, "human_sexuality": 0.25396825396825395, "international_law": 0.2702702702702703, "journalism": 0.32558139534883723, "jurisprudence": 0.24087591240875914, "legal_and_moral_basis": 0.2523364485981308, "logical": 0.2682926829268293, "machine_learning": 0.2786885245901639, "management": 0.26666666666666666, "marketing": 0.25, "marxist_theory": 0.2275132275132275, "modern_chinese": 0.3017241379310345, "nutrition": 0.2482758620689655, "philosophy": 0.19047619047619047, "professional_accounting": 0.19428571428571428, "professional_law": 0.24644549763033174, "professional_medicine": 0.2154255319148936, "professional_psychology": 0.21120689655172414, "public_relations": 0.27586206896551724, "security_study": 0.25925925925925924, "sociology": 0.24778761061946902, "sports_science": 0.2787878787878788, "traditional_chinese_medicine": 0.2702702702702703, "virology": 0.16568047337278108, "world_history": 0.2546583850931677, "world_religions": 0.30625 } } }, "zbench": { "prompt_1": { "accuracy": 0.36363636363636365 }, "prompt_2": { "accuracy": 0.2727272727272727 }, "prompt_3": { "accuracy": 0.2727272727272727 }, "prompt_4": { "accuracy": 0.2727272727272727 }, "prompt_5": { "accuracy": 0.30303030303030304 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.4340909090909091 }, "prompt_2": { "accuracy": 0.42727272727272725 }, "prompt_3": { "accuracy": 0.42272727272727273 }, "prompt_4": { "accuracy": 0.4590909090909091 }, "prompt_5": { "accuracy": 0.46136363636363636 } }, "ocnli": { "prompt_1": { "accuracy": 0.3454237288135593 }, "prompt_2": { "accuracy": 0.3454237288135593 }, "prompt_3": { "accuracy": 0.34508474576271186 }, "prompt_4": { "accuracy": 0.348135593220339 }, "prompt_5": { "accuracy": 0.3461016949152542 } }, "c3": { "prompt_1": { "accuracy": 0.2715033657442034 }, "prompt_2": { "accuracy": 0.2696335078534031 }, "prompt_3": { "accuracy": 0.2696335078534031 }, "prompt_4": { "accuracy": 0.2726252804786836 }, "prompt_5": { "accuracy": 0.2707554225878833 } }, "dream": { "prompt_1": { "accuracy": 0.9299363057324841 }, "prompt_2": { "accuracy": 0.9294463498285155 }, "prompt_3": { "accuracy": 0.9294463498285155 }, "prompt_4": { "accuracy": 0.9284664380205782 }, "prompt_5": { "accuracy": 0.9279764821166095 } }, "samsum": { "prompt_1": { "rouge1": 0.5431764893139259, "rouge2": 0.29470708142921725, "rougeL": 0.4528347926542214, "avg_rouge": 0.43023945446578815 }, "prompt_2": { "rouge1": 0.542488939942541, "rouge2": 0.2933689952477829, "rougeL": 0.45139235454365445, "avg_rouge": 0.42908342991132614 }, "prompt_3": { "rouge1": 0.5373293668982885, "rouge2": 0.2908203324216751, "rougeL": 0.44950047209810673, "avg_rouge": 0.42588339047269014 }, "prompt_4": { "rouge1": 0.5401317921433638, "rouge2": 0.28908707215882196, "rougeL": 0.44795067378763737, "avg_rouge": 0.42572317936327436 }, "prompt_5": { "rouge1": 0.5430637681899185, "rouge2": 0.292544342841456, "rougeL": 0.4526876943035778, "avg_rouge": 0.42943193511165073 } }, "dialogsum": { "prompt_1": { "rouge1": 0.2923360623216347, "rouge2": 0.10186037739881519, "rougeL": 0.22620543198594276, "avg_rouge": 0.20680062390213091 }, "prompt_2": { "rouge1": 0.3058212189179107, "rouge2": 0.11173668321767392, "rougeL": 0.2398256417068377, "avg_rouge": 0.2191278479474741 }, "prompt_3": { "rouge1": 0.30430152353756323, "rouge2": 0.11015725862414502, "rougeL": 0.2378957718944913, "avg_rouge": 0.2174515180187332 }, "prompt_4": { "rouge1": 0.2568407137049432, "rouge2": 0.08122523878099604, "rougeL": 0.19502090677295342, "avg_rouge": 0.17769561975296422 }, "prompt_5": { "rouge1": 0.3030613159708064, "rouge2": 0.10838718211190466, "rougeL": 0.23768183221016057, "avg_rouge": 0.21637677676429054 } }, "sst2": { "prompt_1": { "accuracy": 0.9621559633027523 }, "prompt_2": { "accuracy": 0.9610091743119266 }, "prompt_3": { "accuracy": 0.9610091743119266 }, "prompt_4": { "accuracy": 0.9621559633027523 }, "prompt_5": { "accuracy": 0.9621559633027523 } }, "cola": { "prompt_1": { "accuracy": 0.8465963566634708 }, "prompt_2": { "accuracy": 0.8475551294343241 }, "prompt_3": { "accuracy": 0.850431447746884 }, "prompt_4": { "accuracy": 0.8437200383509108 }, "prompt_5": { "accuracy": 0.8465963566634708 } }, "qqp": { "prompt_1": { "accuracy": 0.841 }, "prompt_2": { "accuracy": 0.8655 }, "prompt_3": { "accuracy": 0.832 }, "prompt_4": { "accuracy": 0.866 }, "prompt_5": { "accuracy": 0.8695 } }, "mnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } }, "qnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "wnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 1.0 }, "prompt_5": { "accuracy": 1.0 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "flan_t5_small": { "model_size": "0.06B", "model_link": "https://huggingface.co/google/flan-t5-small", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.4852941176470588, "language_acc": { "Spanish": 0.5714285714285714, "English": 0.6621848739495798, "Chinese": 0.31596638655462184, "Vietnamese": 0.3915966386554622 }, "consistency_score_2": 0.4644257703081232, "consistency_score_3": 0.2743697478991597, "consistency_score_4": 0.16974789915966387, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.6714285714285714, "Spanish,Chinese": 0.3773109243697479, "Spanish,Vietnamese": 0.5310924369747899, "English,Chinese": 0.3815126050420168, "English,Vietnamese": 0.5, "Chinese,Vietnamese": 0.32521008403361346 }, "3_combine": { "Spanish,English,Chinese": 0.2781512605042017, "Spanish,English,Vietnamese": 0.39663865546218485, "Spanish,Chinese,Vietnamese": 0.21008403361344538, "English,Chinese,Vietnamese": 0.21260504201680672 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.16974789915966387 } }, "AC3_2": 0.47463067215696914, "AC3_3": 0.3505498437851925, "AC3_4": 0.2515186959582823 }, "prompt_2": { "overall_acc": 0.486764705882353, "language_acc": { "Spanish": 0.5739495798319327, "English": 0.6495798319327731, "Chinese": 0.3243697478991597, "Vietnamese": 0.39915966386554624 }, "consistency_score_2": 0.4558823529411764, "consistency_score_3": 0.26701680672268907, "consistency_score_4": 0.16554621848739495, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.6974789915966386, "Spanish,Chinese": 0.38907563025210085, "Spanish,Vietnamese": 0.5092436974789916, "English,Chinese": 0.37142857142857144, "English,Vietnamese": 0.4865546218487395, "Chinese,Vietnamese": 0.2815126050420168 }, "3_combine": { "Spanish,English,Chinese": 0.2857142857142857, "Spanish,English,Vietnamese": 0.39327731092436974, "Spanish,Chinese,Vietnamese": 0.1957983193277311, "English,Chinese,Vietnamese": 0.19327731092436976 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.16554621848739495 } }, "AC3_2": 0.4708176561857148, "AC3_3": 0.34485949893778933, "AC3_4": 0.24706640140192743 }, "prompt_3": { "overall_acc": 0.475, "language_acc": { "Spanish": 0.5563025210084034, "English": 0.6386554621848739, "Chinese": 0.32605042016806723, "Vietnamese": 0.37899159663865545 }, "consistency_score_2": 0.4414565826330532, "consistency_score_3": 0.25168067226890756, "consistency_score_4": 0.15378151260504203, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.66890756302521, "Spanish,Chinese": 0.3420168067226891, "Spanish,Vietnamese": 0.5100840336134453, "English,Chinese": 0.33949579831932775, "English,Vietnamese": 0.48823529411764705, "Chinese,Vietnamese": 0.3 }, "3_combine": { "Spanish,English,Chinese": 0.2504201680672269, "Spanish,English,Vietnamese": 0.38823529411764707, "Spanish,Chinese,Vietnamese": 0.18319327731092436, "English,Chinese,Vietnamese": 0.18487394957983194 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.15378151260504203 } }, "AC3_2": 0.45761442648021133, "AC3_3": 0.32902572993448503, "AC3_4": 0.2323421316035433 }, "prompt_4": { "overall_acc": 0.4859243697478992, "language_acc": { "Spanish": 0.5756302521008403, "English": 0.6495798319327731, "Chinese": 0.31932773109243695, "Vietnamese": 0.39915966386554624 }, "consistency_score_2": 0.4598039215686274, "consistency_score_3": 0.2710084033613445, "consistency_score_4": 0.16554621848739495, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.6957983193277311, "Spanish,Chinese": 0.37478991596638656, "Spanish,Vietnamese": 0.5218487394957984, "English,Chinese": 0.37815126050420167, "English,Vietnamese": 0.5042016806722689, "Chinese,Vietnamese": 0.28403361344537814 }, "3_combine": { "Spanish,English,Chinese": 0.2823529411764706, "Spanish,English,Vietnamese": 0.40588235294117647, "Spanish,Chinese,Vietnamese": 0.1957983193277311, "English,Chinese,Vietnamese": 0.2 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.16554621848739495 } }, "AC3_2": 0.47250343005217904, "AC3_3": 0.3479558350774224, "AC3_4": 0.24695801567402034 }, "prompt_5": { "overall_acc": 0.4915966386554622, "language_acc": { "Spanish": 0.565546218487395, "English": 0.6823529411764706, "Chinese": 0.3134453781512605, "Vietnamese": 0.4050420168067227 }, "consistency_score_2": 0.457703081232493, "consistency_score_3": 0.2619747899159664, "consistency_score_4": 0.17058823529411765, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.673109243697479, "Spanish,Chinese": 0.3420168067226891, "Spanish,Vietnamese": 0.5092436974789916, "English,Chinese": 0.3596638655462185, "English,Vietnamese": 0.48403361344537815, "Chinese,Vietnamese": 0.37815126050420167 }, "3_combine": { "Spanish,English,Chinese": 0.2529411764705882, "Spanish,English,Vietnamese": 0.37899159663865545, "Spanish,Chinese,Vietnamese": 0.20756302521008405, "English,Chinese,Vietnamese": 0.20840336134453782 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.17058823529411765 } }, "AC3_2": 0.4740447963873758, "AC3_3": 0.34180151007639403, "AC3_4": 0.253284562517737 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.26952380952380955, "language_acc": { "Filipino": 0.24, "Vietnamese": 0.3, "Chinese": 0.29333333333333333, "Spanish": 0.29333333333333333, "Malay": 0.23333333333333334, "Indonesian": 0.26, "English": 0.26666666666666666 }, "consistency_score_2": 0.45269841269841266, "consistency_score_3": 0.2617142857142857, "consistency_score_4": 0.17485714285714288, "consistency_score_5": 0.1292063492063492, "consistency_score_6": 0.10285714285714287, "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.41333333333333333, "Filipino,Chinese": 0.38, "Filipino,Spanish": 0.43333333333333335, "Filipino,Malay": 0.47333333333333333, "Filipino,Indonesian": 0.43333333333333335, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.5533333333333333, "Vietnamese,Spanish": 0.43333333333333335, "Vietnamese,Malay": 0.5, "Vietnamese,Indonesian": 0.5066666666666667, "Vietnamese,English": 0.38666666666666666, "Chinese,Spanish": 0.4266666666666667, "Chinese,Malay": 0.4266666666666667, "Chinese,Indonesian": 0.4266666666666667, "Chinese,English": 0.43333333333333335, "Spanish,Malay": 0.46, "Spanish,Indonesian": 0.3933333333333333, "Spanish,English": 0.48, "Malay,Indonesian": 0.6333333333333333, "Malay,English": 0.4266666666666667, "Indonesian,English": 0.42 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.26666666666666666, "Filipino,Vietnamese,Spanish": 0.23333333333333334, "Filipino,Vietnamese,Malay": 0.28, "Filipino,Vietnamese,Indonesian": 0.28, "Filipino,Vietnamese,English": 0.24, "Filipino,Chinese,Spanish": 0.21333333333333335, "Filipino,Chinese,Malay": 0.24, "Filipino,Chinese,Indonesian": 0.22666666666666666, "Filipino,Chinese,English": 0.24, "Filipino,Spanish,Malay": 0.25333333333333335, "Filipino,Spanish,Indonesian": 0.22666666666666666, "Filipino,Spanish,English": 0.2733333333333333, "Filipino,Malay,Indonesian": 0.34, "Filipino,Malay,English": 0.26666666666666666, "Filipino,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish": 0.2866666666666667, "Vietnamese,Chinese,Malay": 0.30666666666666664, "Vietnamese,Chinese,Indonesian": 0.3, "Vietnamese,Chinese,English": 0.25333333333333335, "Vietnamese,Spanish,Malay": 0.28, "Vietnamese,Spanish,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Malay,English": 0.22, "Vietnamese,Indonesian,English": 0.22, "Chinese,Spanish,Malay": 0.23333333333333334, "Chinese,Spanish,Indonesian": 0.22666666666666666, "Chinese,Spanish,English": 0.25333333333333335, "Chinese,Malay,Indonesian": 0.31333333333333335, "Chinese,Malay,English": 0.21333333333333335, "Chinese,Indonesian,English": 0.22666666666666666, "Spanish,Malay,Indonesian": 0.31333333333333335, "Spanish,Malay,English": 0.25333333333333335, "Spanish,Indonesian,English": 0.24666666666666667, "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Malay,English": 0.16, "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.14, "Filipino,Chinese,Spanish,English": 0.16, "Filipino,Chinese,Malay,Indonesian": 0.19333333333333333, "Filipino,Chinese,Malay,English": 0.16, "Filipino,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Spanish,Malay,English": 0.18, "Filipino,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Spanish,English": 0.18, "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Spanish,Malay,English": 0.16, "Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, "Vietnamese,Malay,Indonesian,English": 0.16666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.18, "Chinese,Spanish,Malay,English": 0.15333333333333332, "Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Chinese,Malay,Indonesian,English": 0.16666666666666666, "Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Spanish,Malay,English": 0.12, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12, "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.12, "Filipino,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Vietnamese,Chinese,Spanish,Malay,English": 0.12, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, "AC3_2": 0.33788215589251513, "AC3_3": 0.2655616451889597, "AC3_4": 0.21210703565124653, "AC3_5": 0.17467546249789664, "AC3_6": 0.14889294844375728, "AC3_7": 0.1311586452394697 }, "prompt_2": { "overall_acc": 0.259047619047619, "language_acc": { "Filipino": 0.24, "Vietnamese": 0.26, "Chinese": 0.25333333333333335, "Spanish": 0.2866666666666667, "Malay": 0.22, "Indonesian": 0.24666666666666667, "English": 0.30666666666666664 }, "consistency_score_2": 0.43777777777777777, "consistency_score_3": 0.2558095238095238, "consistency_score_4": 0.1794285714285714, "consistency_score_5": 0.14063492063492064, "consistency_score_6": 0.11904761904761904, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5133333333333333, "Filipino,Chinese": 0.26666666666666666, "Filipino,Spanish": 0.5066666666666667, "Filipino,Malay": 0.47333333333333333, "Filipino,Indonesian": 0.5, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.3333333333333333, "Vietnamese,Spanish": 0.44666666666666666, "Vietnamese,Malay": 0.4866666666666667, "Vietnamese,Indonesian": 0.5, "Vietnamese,English": 0.38, "Chinese,Spanish": 0.24, "Chinese,Malay": 0.3466666666666667, "Chinese,Indonesian": 0.38, "Chinese,English": 0.3466666666666667, "Spanish,Malay": 0.47333333333333333, "Spanish,Indonesian": 0.4866666666666667, "Spanish,English": 0.5333333333333333, "Malay,Indonesian": 0.6733333333333333, "Malay,English": 0.44, "Indonesian,English": 0.4 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.18666666666666668, "Filipino,Vietnamese,Spanish": 0.31333333333333335, "Filipino,Vietnamese,Malay": 0.30666666666666664, "Filipino,Vietnamese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,English": 0.29333333333333333, "Filipino,Chinese,Spanish": 0.16666666666666666, "Filipino,Chinese,Malay": 0.16, "Filipino,Chinese,Indonesian": 0.18, "Filipino,Chinese,English": 0.17333333333333334, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.3333333333333333, "Filipino,Malay,Indonesian": 0.36666666666666664, "Filipino,Malay,English": 0.28, "Filipino,Indonesian,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish": 0.16, "Vietnamese,Chinese,Malay": 0.20666666666666667, "Vietnamese,Chinese,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,English": 0.17333333333333334, "Vietnamese,Spanish,Malay": 0.29333333333333333, "Vietnamese,Spanish,Indonesian": 0.3, "Vietnamese,Spanish,English": 0.28, "Vietnamese,Malay,Indonesian": 0.38, "Vietnamese,Malay,English": 0.24, "Vietnamese,Indonesian,English": 0.24, "Chinese,Spanish,Malay": 0.16666666666666666, "Chinese,Spanish,Indonesian": 0.16666666666666666, "Chinese,Spanish,English": 0.18666666666666668, "Chinese,Malay,Indonesian": 0.26666666666666666, "Chinese,Malay,English": 0.18666666666666668, "Chinese,Indonesian,English": 0.18666666666666668, "Spanish,Malay,Indonesian": 0.36666666666666664, "Spanish,Malay,English": 0.30666666666666664, "Spanish,Indonesian,English": 0.29333333333333333, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.14, "Filipino,Vietnamese,Chinese,Malay": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,English": 0.24, "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay": 0.12666666666666668, "Filipino,Chinese,Spanish,Indonesian": 0.14, "Filipino,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.16, "Filipino,Chinese,Malay,English": 0.12666666666666668, "Filipino,Chinese,Indonesian,English": 0.13333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Spanish,Malay,English": 0.23333333333333334, "Filipino,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.14, "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, "Vietnamese,Chinese,Malay,Indonesian": 0.18, "Vietnamese,Chinese,Malay,English": 0.13333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.12666666666666668, "Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.2, "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Chinese,Spanish,Malay,English": 0.14, "Chinese,Spanish,Indonesian,English": 0.14, "Chinese,Malay,Indonesian,English": 0.16666666666666666, "Spanish,Malay,Indonesian,English": 0.25333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.12, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.12, "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.32549126798669864, "AC3_3": 0.2574183886891454, "AC3_4": 0.2120094327430101, "AC3_5": 0.18230038953039385, "AC3_6": 0.16312822353847933, "AC3_7": 0.15111111106979166 }, "prompt_3": { "overall_acc": 0.2733333333333333, "language_acc": { "Filipino": 0.28, "Vietnamese": 0.28, "Chinese": 0.24666666666666667, "Spanish": 0.3, "Malay": 0.26666666666666666, "Indonesian": 0.25333333333333335, "English": 0.2866666666666667 }, "consistency_score_2": 0.3946031746031746, "consistency_score_3": 0.2085714285714286, "consistency_score_4": 0.13657142857142857, "consistency_score_5": 0.10126984126984126, "consistency_score_6": 0.08095238095238096, "consistency_score_7": 0.06666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.3333333333333333, "Filipino,Chinese": 0.30666666666666664, "Filipino,Spanish": 0.4666666666666667, "Filipino,Malay": 0.47333333333333333, "Filipino,Indonesian": 0.54, "Filipino,English": 0.4066666666666667, "Vietnamese,Chinese": 0.30666666666666664, "Vietnamese,Spanish": 0.35333333333333333, "Vietnamese,Malay": 0.37333333333333335, "Vietnamese,Indonesian": 0.35333333333333333, "Vietnamese,English": 0.2866666666666667, "Chinese,Spanish": 0.29333333333333333, "Chinese,Malay": 0.31333333333333335, "Chinese,Indonesian": 0.34, "Chinese,English": 0.30666666666666664, "Spanish,Malay": 0.44666666666666666, "Spanish,Indonesian": 0.46, "Spanish,English": 0.47333333333333333, "Malay,Indonesian": 0.6533333333333333, "Malay,English": 0.38, "Indonesian,English": 0.42 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.14, "Filipino,Vietnamese,Spanish": 0.18, "Filipino,Vietnamese,Malay": 0.20666666666666667, "Filipino,Vietnamese,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,English": 0.16, "Filipino,Chinese,Spanish": 0.18666666666666668, "Filipino,Chinese,Malay": 0.18666666666666668, "Filipino,Chinese,Indonesian": 0.20666666666666667, "Filipino,Chinese,English": 0.16, "Filipino,Spanish,Malay": 0.26666666666666666, "Filipino,Spanish,Indonesian": 0.32, "Filipino,Spanish,English": 0.24666666666666667, "Filipino,Malay,Indonesian": 0.38, "Filipino,Malay,English": 0.22, "Filipino,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish": 0.13333333333333333, "Vietnamese,Chinese,Malay": 0.14, "Vietnamese,Chinese,Indonesian": 0.13333333333333333, "Vietnamese,Chinese,English": 0.12, "Vietnamese,Spanish,Malay": 0.20666666666666667, "Vietnamese,Spanish,Indonesian": 0.20666666666666667, "Vietnamese,Spanish,English": 0.17333333333333334, "Vietnamese,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Malay,English": 0.15333333333333332, "Vietnamese,Indonesian,English": 0.16, "Chinese,Spanish,Malay": 0.19333333333333333, "Chinese,Spanish,Indonesian": 0.18666666666666668, "Chinese,Spanish,English": 0.18, "Chinese,Malay,Indonesian": 0.23333333333333334, "Chinese,Malay,English": 0.16, "Chinese,Indonesian,English": 0.18, "Spanish,Malay,Indonesian": 0.34, "Spanish,Malay,English": 0.26, "Spanish,Indonesian,English": 0.26, "Malay,Indonesian,English": 0.2866666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,English": 0.08666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.14, "Filipino,Vietnamese,Spanish,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Spanish,English": 0.12666666666666668, "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Malay,English": 0.12, "Filipino,Vietnamese,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Chinese,Spanish,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian": 0.16666666666666666, "Filipino,Chinese,Malay,English": 0.11333333333333333, "Filipino,Chinese,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Spanish,Malay,English": 0.16666666666666666, "Filipino,Spanish,Indonesian,English": 0.18666666666666668, "Filipino,Malay,Indonesian,English": 0.18, "Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.1, "Vietnamese,Chinese,Spanish,English": 0.08666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.12, "Vietnamese,Chinese,Malay,English": 0.09333333333333334, "Vietnamese,Chinese,Indonesian,English": 0.08666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.18, "Vietnamese,Spanish,Malay,English": 0.12666666666666668, "Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, "Vietnamese,Malay,Indonesian,English": 0.14, "Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Chinese,Spanish,Malay,English": 0.13333333333333333, "Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Chinese,Malay,Indonesian,English": 0.12666666666666668, "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Malay,English": 0.08, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Vietnamese,Chinese,Spanish,Malay,English": 0.08, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, "Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.1, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 } }, "AC3_2": 0.3229594422837141, "AC3_3": 0.23660079046473698, "AC3_4": 0.18213754642396737, "AC3_5": 0.1477853106950121, "AC3_6": 0.12491039422997602, "AC3_7": 0.10718954245213379 }, "prompt_4": { "overall_acc": 0.26571428571428574, "language_acc": { "Filipino": 0.26666666666666666, "Vietnamese": 0.29333333333333333, "Chinese": 0.23333333333333334, "Spanish": 0.2866666666666667, "Malay": 0.21333333333333335, "Indonesian": 0.26, "English": 0.30666666666666664 }, "consistency_score_2": 0.42920634920634915, "consistency_score_3": 0.24285714285714285, "consistency_score_4": 0.1660952380952381, "consistency_score_5": 0.12888888888888891, "consistency_score_6": 0.10952380952380951, "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.47333333333333333, "Filipino,Chinese": 0.2866666666666667, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.44, "Filipino,Indonesian": 0.4866666666666667, "Filipino,English": 0.5266666666666666, "Vietnamese,Chinese": 0.35333333333333333, "Vietnamese,Spanish": 0.44666666666666666, "Vietnamese,Malay": 0.4866666666666667, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.38, "Chinese,Spanish": 0.24, "Chinese,Malay": 0.36666666666666664, "Chinese,Indonesian": 0.38666666666666666, "Chinese,English": 0.3333333333333333, "Spanish,Malay": 0.41333333333333333, "Spanish,Indonesian": 0.44666666666666666, "Spanish,English": 0.5, "Malay,Indonesian": 0.66, "Malay,English": 0.43333333333333335, "Indonesian,English": 0.4066666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.16, "Filipino,Vietnamese,Spanish": 0.2733333333333333, "Filipino,Vietnamese,Malay": 0.28, "Filipino,Vietnamese,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,English": 0.3, "Filipino,Chinese,Spanish": 0.15333333333333332, "Filipino,Chinese,Malay": 0.15333333333333332, "Filipino,Chinese,Indonesian": 0.18666666666666668, "Filipino,Chinese,English": 0.18, "Filipino,Spanish,Malay": 0.23333333333333334, "Filipino,Spanish,Indonesian": 0.26, "Filipino,Spanish,English": 0.3333333333333333, "Filipino,Malay,Indonesian": 0.36, "Filipino,Malay,English": 0.3, "Filipino,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish": 0.16, "Vietnamese,Chinese,Malay": 0.20666666666666667, "Vietnamese,Chinese,Indonesian": 0.22, "Vietnamese,Chinese,English": 0.16, "Vietnamese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,English": 0.26666666666666666, "Vietnamese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Malay,English": 0.24666666666666667, "Vietnamese,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay": 0.14666666666666667, "Chinese,Spanish,Indonesian": 0.16, "Chinese,Spanish,English": 0.16, "Chinese,Malay,Indonesian": 0.26666666666666666, "Chinese,Malay,English": 0.19333333333333333, "Chinese,Indonesian,English": 0.18, "Spanish,Malay,Indonesian": 0.31333333333333335, "Spanish,Malay,English": 0.25333333333333335, "Spanish,Indonesian,English": 0.26, "Malay,Indonesian,English": 0.32666666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,English": 0.12, "Filipino,Vietnamese,Spanish,Malay": 0.18, "Filipino,Vietnamese,Spanish,Indonesian": 0.2, "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay": 0.10666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian": 0.15333333333333332, "Filipino,Chinese,Malay,English": 0.12666666666666668, "Filipino,Chinese,Indonesian,English": 0.14, "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Spanish,Malay,English": 0.2, "Filipino,Spanish,Indonesian,English": 0.21333333333333335, "Filipino,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.14666666666666667, "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, "Vietnamese,Chinese,Malay,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Malay,English": 0.12666666666666668, "Vietnamese,Chinese,Indonesian,English": 0.12666666666666668, "Vietnamese,Spanish,Malay,Indonesian": 0.22, "Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Chinese,Spanish,Malay,English": 0.12, "Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Chinese,Malay,Indonesian,English": 0.16, "Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.1, "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, "Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, "AC3_2": 0.32822815370855946, "AC3_3": 0.25377207057610424, "AC3_4": 0.20441363660273362, "AC3_5": 0.17358004822632528, "AC3_6": 0.1551124002487282, "AC3_7": 0.1453124999602661 }, "prompt_5": { "overall_acc": 0.2733333333333333, "language_acc": { "Filipino": 0.28, "Vietnamese": 0.2866666666666667, "Chinese": 0.2866666666666667, "Spanish": 0.26, "Malay": 0.28, "Indonesian": 0.23333333333333334, "English": 0.2866666666666667 }, "consistency_score_2": 0.45269841269841266, "consistency_score_3": 0.25866666666666666, "consistency_score_4": 0.17314285714285718, "consistency_score_5": 0.1298412698412698, "consistency_score_6": 0.10380952380952381, "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4533333333333333, "Filipino,Chinese": 0.30666666666666664, "Filipino,Spanish": 0.49333333333333335, "Filipino,Malay": 0.49333333333333335, "Filipino,Indonesian": 0.5133333333333333, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.5133333333333333, "Vietnamese,Spanish": 0.43333333333333335, "Vietnamese,Malay": 0.5066666666666667, "Vietnamese,Indonesian": 0.44666666666666666, "Vietnamese,English": 0.4, "Chinese,Spanish": 0.37333333333333335, "Chinese,Malay": 0.3333333333333333, "Chinese,Indonesian": 0.36, "Chinese,English": 0.34, "Spanish,Malay": 0.5066666666666667, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.5066666666666667, "Malay,Indonesian": 0.6933333333333334, "Malay,English": 0.4266666666666667, "Indonesian,English": 0.44666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.21333333333333335, "Filipino,Vietnamese,Spanish": 0.2866666666666667, "Filipino,Vietnamese,Malay": 0.2866666666666667, "Filipino,Vietnamese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,English": 0.25333333333333335, "Filipino,Chinese,Spanish": 0.19333333333333333, "Filipino,Chinese,Malay": 0.15333333333333332, "Filipino,Chinese,Indonesian": 0.16, "Filipino,Chinese,English": 0.18, "Filipino,Spanish,Malay": 0.3, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.32, "Filipino,Malay,Indonesian": 0.38666666666666666, "Filipino,Malay,English": 0.2866666666666667, "Filipino,Indonesian,English": 0.3, "Vietnamese,Chinese,Spanish": 0.24666666666666667, "Vietnamese,Chinese,Malay": 0.24, "Vietnamese,Chinese,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,English": 0.20666666666666667, "Vietnamese,Spanish,Malay": 0.29333333333333333, "Vietnamese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,English": 0.26666666666666666, "Vietnamese,Malay,Indonesian": 0.36666666666666664, "Vietnamese,Malay,English": 0.26, "Vietnamese,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay": 0.2, "Chinese,Spanish,Indonesian": 0.20666666666666667, "Chinese,Spanish,English": 0.19333333333333333, "Chinese,Malay,Indonesian": 0.25333333333333335, "Chinese,Malay,English": 0.16, "Chinese,Indonesian,English": 0.17333333333333334, "Spanish,Malay,Indonesian": 0.3933333333333333, "Spanish,Malay,English": 0.29333333333333333, "Spanish,Indonesian,English": 0.32666666666666666, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,English": 0.2, "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.13333333333333333, "Filipino,Chinese,Spanish,English": 0.12666666666666668, "Filipino,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Chinese,Malay,English": 0.10666666666666667, "Filipino,Chinese,Indonesian,English": 0.11333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.26, "Filipino,Spanish,Malay,English": 0.22666666666666666, "Filipino,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Malay,English": 0.12666666666666668, "Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.16, "Chinese,Spanish,Malay,English": 0.11333333333333333, "Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Chinese,Malay,Indonesian,English": 0.13333333333333333, "Spanish,Malay,Indonesian,English": 0.26 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Malay,English": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Chinese,Spanish,Malay,English": 0.09333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Vietnamese,Chinese,Spanish,Malay,English": 0.1, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, "AC3_2": 0.34085993290743144, "AC3_3": 0.26579782785312905, "AC3_4": 0.21199658698323456, "AC3_5": 0.1760524933946536, "AC3_6": 0.15047138043148275, "AC3_7": 0.131604938235048 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.27435064935064934, "language_acc": { "Indonesian": 0.26704545454545453, "English": 0.2897727272727273, "Filipino": 0.2727272727272727, "Spanish": 0.2897727272727273, "Chinese": 0.23863636363636365, "Malay": 0.2727272727272727, "Vietnamese": 0.2897727272727273 }, "consistency_score_2": 0.3890692640692641, "consistency_score_3": 0.1931818181818182, "consistency_score_4": 0.1099025974025974, "consistency_score_5": 0.06737012987012989, "consistency_score_6": 0.04301948051948052, "consistency_score_7": 0.028409090909090908, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.39204545454545453, "Indonesian,Filipino": 0.4772727272727273, "Indonesian,Spanish": 0.3977272727272727, "Indonesian,Chinese": 0.26136363636363635, "Indonesian,Malay": 0.5568181818181818, "Indonesian,Vietnamese": 0.32954545454545453, "English,Filipino": 0.4318181818181818, "English,Spanish": 0.5397727272727273, "English,Chinese": 0.26136363636363635, "English,Malay": 0.4318181818181818, "English,Vietnamese": 0.3522727272727273, "Filipino,Spanish": 0.4772727272727273, "Filipino,Chinese": 0.32954545454545453, "Filipino,Malay": 0.48295454545454547, "Filipino,Vietnamese": 0.38636363636363635, "Spanish,Chinese": 0.30113636363636365, "Spanish,Malay": 0.42045454545454547, "Spanish,Vietnamese": 0.32954545454545453, "Chinese,Malay": 0.3125, "Chinese,Vietnamese": 0.35795454545454547, "Malay,Vietnamese": 0.3409090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.25, "Indonesian,English,Spanish": 0.23863636363636365, "Indonesian,English,Chinese": 0.11363636363636363, "Indonesian,English,Malay": 0.2727272727272727, "Indonesian,English,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish": 0.25, "Indonesian,Filipino,Chinese": 0.1534090909090909, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.17613636363636365, "Indonesian,Spanish,Chinese": 0.125, "Indonesian,Spanish,Malay": 0.2897727272727273, "Indonesian,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,Chinese,Malay": 0.17613636363636365, "Indonesian,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish": 0.29545454545454547, "English,Filipino,Chinese": 0.14772727272727273, "English,Filipino,Malay": 0.2727272727272727, "English,Filipino,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese": 0.16477272727272727, "English,Spanish,Malay": 0.26704545454545453, "English,Spanish,Vietnamese": 0.19886363636363635, "English,Chinese,Malay": 0.13636363636363635, "English,Chinese,Vietnamese": 0.1534090909090909, "English,Malay,Vietnamese": 0.17613636363636365, "Filipino,Spanish,Chinese": 0.1534090909090909, "Filipino,Spanish,Malay": 0.2840909090909091, "Filipino,Spanish,Vietnamese": 0.18181818181818182, "Filipino,Chinese,Malay": 0.17045454545454544, "Filipino,Chinese,Vietnamese": 0.18181818181818182, "Filipino,Malay,Vietnamese": 0.19886363636363635, "Spanish,Chinese,Malay": 0.1534090909090909, "Spanish,Chinese,Vietnamese": 0.14772727272727273, "Spanish,Malay,Vietnamese": 0.1590909090909091, "Chinese,Malay,Vietnamese": 0.14772727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1590909090909091, "Indonesian,English,Filipino,Chinese": 0.09090909090909091, "Indonesian,English,Filipino,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese": 0.08522727272727272, "Indonesian,English,Spanish,Malay": 0.18181818181818182, "Indonesian,English,Spanish,Vietnamese": 0.08522727272727272, "Indonesian,English,Chinese,Malay": 0.07954545454545454, "Indonesian,English,Chinese,Vietnamese": 0.0625, "Indonesian,English,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.08522727272727272, "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, "Indonesian,Filipino,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,Filipino,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, "Indonesian,Spanish,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese": 0.10795454545454546, "English,Filipino,Spanish,Malay": 0.1875, "English,Filipino,Spanish,Vietnamese": 0.125, "English,Filipino,Chinese,Malay": 0.09090909090909091, "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, "English,Filipino,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay": 0.10227272727272728, "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, "English,Spanish,Malay,Vietnamese": 0.10795454545454546, "English,Chinese,Malay,Vietnamese": 0.07954545454545454, "Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, "Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.0625, "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay": 0.0625, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Spanish,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, "English,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, "English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, "English,Spanish,Chinese,Malay,Vietnamese": 0.0625, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 } }, "AC3_2": 0.3217913814900218, "AC3_3": 0.22672032823433533, "AC3_4": 0.15693738028056856, "AC3_5": 0.10817626550802964, "AC3_6": 0.07437639084911933, "AC3_7": 0.05148671701930909 }, "prompt_2": { "overall_acc": 0.2792207792207792, "language_acc": { "Indonesian": 0.2897727272727273, "English": 0.29545454545454547, "Filipino": 0.25, "Spanish": 0.2784090909090909, "Chinese": 0.2897727272727273, "Malay": 0.2840909090909091, "Vietnamese": 0.26704545454545453 }, "consistency_score_2": 0.38771645021645024, "consistency_score_3": 0.18490259740259737, "consistency_score_4": 0.09724025974025972, "consistency_score_5": 0.054653679653679677, "consistency_score_6": 0.03327922077922078, "consistency_score_7": 0.022727272727272728, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.38636363636363635, "Indonesian,Filipino": 0.5454545454545454, "Indonesian,Spanish": 0.4147727272727273, "Indonesian,Chinese": 0.2784090909090909, "Indonesian,Malay": 0.6136363636363636, "Indonesian,Vietnamese": 0.375, "English,Filipino": 0.44886363636363635, "English,Spanish": 0.5, "English,Chinese": 0.2784090909090909, "English,Malay": 0.4318181818181818, "English,Vietnamese": 0.36363636363636365, "Filipino,Spanish": 0.45454545454545453, "Filipino,Chinese": 0.19318181818181818, "Filipino,Malay": 0.5511363636363636, "Filipino,Vietnamese": 0.38636363636363635, "Spanish,Chinese": 0.24431818181818182, "Spanish,Malay": 0.4772727272727273, "Spanish,Vietnamese": 0.32386363636363635, "Chinese,Malay": 0.3068181818181818, "Chinese,Vietnamese": 0.22727272727272727, "Malay,Vietnamese": 0.3409090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.26704545454545453, "Indonesian,English,Spanish": 0.23295454545454544, "Indonesian,English,Chinese": 0.11363636363636363, "Indonesian,English,Malay": 0.2727272727272727, "Indonesian,English,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish": 0.2784090909090909, "Indonesian,Filipino,Chinese": 0.125, "Indonesian,Filipino,Malay": 0.39204545454545453, "Indonesian,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Chinese": 0.10227272727272728, "Indonesian,Spanish,Malay": 0.3181818181818182, "Indonesian,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,Chinese,Malay": 0.1875, "Indonesian,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Spanish": 0.2556818181818182, "English,Filipino,Chinese": 0.11363636363636363, "English,Filipino,Malay": 0.26704545454545453, "English,Filipino,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese": 0.125, "English,Spanish,Malay": 0.26704545454545453, "English,Spanish,Vietnamese": 0.18181818181818182, "English,Chinese,Malay": 0.1534090909090909, "English,Chinese,Vietnamese": 0.09090909090909091, "English,Malay,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Chinese": 0.09090909090909091, "Filipino,Spanish,Malay": 0.30113636363636365, "Filipino,Spanish,Vietnamese": 0.17045454545454544, "Filipino,Chinese,Malay": 0.125, "Filipino,Chinese,Vietnamese": 0.07386363636363637, "Filipino,Malay,Vietnamese": 0.20454545454545456, "Spanish,Chinese,Malay": 0.125, "Spanish,Chinese,Vietnamese": 0.07954545454545454, "Spanish,Malay,Vietnamese": 0.1875, "Chinese,Malay,Vietnamese": 0.08522727272727272 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.16477272727272727, "Indonesian,English,Filipino,Chinese": 0.07386363636363637, "Indonesian,English,Filipino,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese": 0.056818181818181816, "Indonesian,English,Spanish,Malay": 0.17045454545454544, "Indonesian,English,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Chinese,Malay": 0.09659090909090909, "Indonesian,English,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Chinese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Malay": 0.20454545454545456, "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, "Indonesian,Filipino,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Chinese,Malay,Vietnamese": 0.06818181818181818, "English,Filipino,Spanish,Chinese": 0.056818181818181816, "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "English,Filipino,Chinese,Malay": 0.07954545454545454, "English,Filipino,Chinese,Vietnamese": 0.05113636363636364, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.09090909090909091, "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, "English,Spanish,Malay,Vietnamese": 0.11363636363636363, "English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Filipino,Spanish,Chinese,Malay": 0.07386363636363637, "Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Filipino,Spanish,Malay,Vietnamese": 0.125, "Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 } }, "AC3_2": 0.3246437132360101, "AC3_3": 0.2224781164490691, "AC3_4": 0.14424600837328067, "AC3_5": 0.09141425832366948, "AC3_6": 0.059470399711109456, "AC3_7": 0.04203323556770199 }, "prompt_3": { "overall_acc": 0.2857142857142857, "language_acc": { "Indonesian": 0.3068181818181818, "English": 0.3068181818181818, "Filipino": 0.25, "Spanish": 0.2840909090909091, "Chinese": 0.29545454545454547, "Malay": 0.2840909090909091, "Vietnamese": 0.2727272727272727 }, "consistency_score_2": 0.49269480519480524, "consistency_score_3": 0.3073051948051948, "consistency_score_4": 0.20990259740259737, "consistency_score_5": 0.15070346320346323, "consistency_score_6": 0.11201298701298701, "consistency_score_7": 0.08522727272727272, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4602272727272727, "Indonesian,Filipino": 0.6704545454545454, "Indonesian,Spanish": 0.5454545454545454, "Indonesian,Chinese": 0.4090909090909091, "Indonesian,Malay": 0.7159090909090909, "Indonesian,Vietnamese": 0.5, "English,Filipino": 0.4659090909090909, "English,Spanish": 0.5454545454545454, "English,Chinese": 0.32386363636363635, "English,Malay": 0.4659090909090909, "English,Vietnamese": 0.4034090909090909, "Filipino,Spanish": 0.5909090909090909, "Filipino,Chinese": 0.3352272727272727, "Filipino,Malay": 0.6988636363636364, "Filipino,Vietnamese": 0.48295454545454547, "Spanish,Chinese": 0.3125, "Spanish,Malay": 0.6136363636363636, "Spanish,Vietnamese": 0.4431818181818182, "Chinese,Malay": 0.42045454545454547, "Chinese,Vietnamese": 0.42045454545454547, "Malay,Vietnamese": 0.5227272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.3522727272727273, "Indonesian,English,Spanish": 0.3409090909090909, "Indonesian,English,Chinese": 0.20454545454545456, "Indonesian,English,Malay": 0.38636363636363635, "Indonesian,English,Vietnamese": 0.26136363636363635, "Indonesian,Filipino,Spanish": 0.42613636363636365, "Indonesian,Filipino,Chinese": 0.2784090909090909, "Indonesian,Filipino,Malay": 0.5568181818181818, "Indonesian,Filipino,Vietnamese": 0.35795454545454547, "Indonesian,Spanish,Chinese": 0.22727272727272727, "Indonesian,Spanish,Malay": 0.4715909090909091, "Indonesian,Spanish,Vietnamese": 0.3068181818181818, "Indonesian,Chinese,Malay": 0.32386363636363635, "Indonesian,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,Malay,Vietnamese": 0.3977272727272727, "English,Filipino,Spanish": 0.36363636363636365, "English,Filipino,Chinese": 0.17613636363636365, "English,Filipino,Malay": 0.375, "English,Filipino,Vietnamese": 0.2727272727272727, "English,Spanish,Chinese": 0.18181818181818182, "English,Spanish,Malay": 0.3806818181818182, "English,Spanish,Vietnamese": 0.26704545454545453, "English,Chinese,Malay": 0.19886363636363635, "English,Chinese,Vietnamese": 0.17613636363636365, "English,Malay,Vietnamese": 0.2556818181818182, "Filipino,Spanish,Chinese": 0.21022727272727273, "Filipino,Spanish,Malay": 0.48863636363636365, "Filipino,Spanish,Vietnamese": 0.32386363636363635, "Filipino,Chinese,Malay": 0.2840909090909091, "Filipino,Chinese,Vietnamese": 0.2159090909090909, "Filipino,Malay,Vietnamese": 0.3977272727272727, "Spanish,Chinese,Malay": 0.2556818181818182, "Spanish,Chinese,Vietnamese": 0.19318181818181818, "Spanish,Malay,Vietnamese": 0.3409090909090909, "Chinese,Malay,Vietnamese": 0.26136363636363635 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2897727272727273, "Indonesian,English,Filipino,Chinese": 0.1534090909090909, "Indonesian,English,Filipino,Malay": 0.32386363636363635, "Indonesian,English,Filipino,Vietnamese": 0.2159090909090909, "Indonesian,English,Spanish,Chinese": 0.14772727272727273, "Indonesian,English,Spanish,Malay": 0.3125, "Indonesian,English,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,English,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,English,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Malay": 0.39204545454545453, "Indonesian,Filipino,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Chinese,Malay": 0.23863636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Spanish,Chinese,Malay": 0.20454545454545456, "Indonesian,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese": 0.13636363636363635, "English,Filipino,Spanish,Malay": 0.32954545454545453, "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "English,Filipino,Chinese,Malay": 0.1534090909090909, "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, "English,Filipino,Malay,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese,Malay": 0.1590909090909091, "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, "English,Spanish,Malay,Vietnamese": 0.21022727272727273, "English,Chinese,Malay,Vietnamese": 0.13636363636363635, "Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.2840909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.13636363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.18181818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 } }, "AC3_2": 0.36168628030509187, "AC3_3": 0.2961166946432734, "AC3_4": 0.2420102006904895, "AC3_5": 0.19732530329372777, "AC3_6": 0.16093294456595084, "AC3_7": 0.1312910284109955 }, "prompt_4": { "overall_acc": 0.28490259740259744, "language_acc": { "Indonesian": 0.2727272727272727, "English": 0.32386363636363635, "Filipino": 0.2556818181818182, "Spanish": 0.32954545454545453, "Chinese": 0.2727272727272727, "Malay": 0.2897727272727273, "Vietnamese": 0.25 }, "consistency_score_2": 0.39772727272727276, "consistency_score_3": 0.19155844155844154, "consistency_score_4": 0.09805194805194804, "consistency_score_5": 0.05032467532467533, "consistency_score_6": 0.024350649350649345, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3693181818181818, "Indonesian,Filipino": 0.5738636363636364, "Indonesian,Spanish": 0.4034090909090909, "Indonesian,Chinese": 0.26136363636363635, "Indonesian,Malay": 0.6875, "Indonesian,Vietnamese": 0.4318181818181818, "English,Filipino": 0.4147727272727273, "English,Spanish": 0.5, "English,Chinese": 0.30113636363636365, "English,Malay": 0.4318181818181818, "English,Vietnamese": 0.32386363636363635, "Filipino,Spanish": 0.5, "Filipino,Chinese": 0.17045454545454544, "Filipino,Malay": 0.625, "Filipino,Vietnamese": 0.4090909090909091, "Spanish,Chinese": 0.23295454545454544, "Spanish,Malay": 0.4602272727272727, "Spanish,Vietnamese": 0.3465909090909091, "Chinese,Malay": 0.24431818181818182, "Chinese,Vietnamese": 0.2215909090909091, "Malay,Vietnamese": 0.4431818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.23863636363636365, "Indonesian,English,Spanish": 0.2159090909090909, "Indonesian,English,Chinese": 0.10795454545454546, "Indonesian,English,Malay": 0.29545454545454547, "Indonesian,English,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish": 0.2897727272727273, "Indonesian,Filipino,Chinese": 0.09659090909090909, "Indonesian,Filipino,Malay": 0.4659090909090909, "Indonesian,Filipino,Vietnamese": 0.2840909090909091, "Indonesian,Spanish,Chinese": 0.10795454545454546, "Indonesian,Spanish,Malay": 0.32386363636363635, "Indonesian,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,Chinese,Malay": 0.16477272727272727, "Indonesian,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Malay,Vietnamese": 0.3352272727272727, "English,Filipino,Spanish": 0.26704545454545453, "English,Filipino,Chinese": 0.09090909090909091, "English,Filipino,Malay": 0.2784090909090909, "English,Filipino,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese": 0.125, "English,Spanish,Malay": 0.26136363636363635, "English,Spanish,Vietnamese": 0.1875, "English,Chinese,Malay": 0.125, "English,Chinese,Vietnamese": 0.07386363636363637, "English,Malay,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Chinese": 0.08522727272727272, "Filipino,Spanish,Malay": 0.3465909090909091, "Filipino,Spanish,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay": 0.09659090909090909, "Filipino,Chinese,Vietnamese": 0.05113636363636364, "Filipino,Malay,Vietnamese": 0.29545454545454547, "Spanish,Chinese,Malay": 0.10795454545454546, "Spanish,Chinese,Vietnamese": 0.07386363636363637, "Spanish,Malay,Vietnamese": 0.21022727272727273, "Chinese,Malay,Vietnamese": 0.08522727272727272 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1534090909090909, "Indonesian,English,Filipino,Chinese": 0.05113636363636364, "Indonesian,English,Filipino,Malay": 0.20454545454545456, "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese": 0.0625, "Indonesian,English,Spanish,Malay": 0.18181818181818182, "Indonesian,English,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Chinese,Malay": 0.09090909090909091, "Indonesian,English,Chinese,Vietnamese": 0.045454545454545456, "Indonesian,English,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.056818181818181816, "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Chinese,Malay": 0.07386363636363637, "Indonesian,Filipino,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,Spanish,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese": 0.045454545454545456, "English,Filipino,Spanish,Malay": 0.19318181818181818, "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, "English,Filipino,Chinese,Malay": 0.045454545454545456, "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Malay,Vietnamese": 0.13636363636363635, "English,Spanish,Chinese,Malay": 0.06818181818181818, "English,Spanish,Chinese,Vietnamese": 0.03977272727272727, "English,Spanish,Malay,Vietnamese": 0.10795454545454546, "English,Chinese,Malay,Vietnamese": 0.03977272727272727, "Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Chinese,Malay": 0.03977272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Filipino,Malay,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Spanish,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, "AC3_2": 0.3319911360447612, "AC3_3": 0.22908692665383723, "AC3_4": 0.14589331817400056, "AC3_5": 0.08553976287502078, "AC3_6": 0.044866550757554545, "AC3_7": 0.021855541711178424 }, "prompt_5": { "overall_acc": 0.2930194805194805, "language_acc": { "Indonesian": 0.3181818181818182, "English": 0.35795454545454547, "Filipino": 0.25, "Spanish": 0.2897727272727273, "Chinese": 0.2727272727272727, "Malay": 0.26704545454545453, "Vietnamese": 0.29545454545454547 }, "consistency_score_2": 0.35633116883116883, "consistency_score_3": 0.16201298701298697, "consistency_score_4": 0.08798701298701296, "consistency_score_5": 0.05330086580086579, "consistency_score_6": 0.034090909090909095, "consistency_score_7": 0.022727272727272728, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.3977272727272727, "Indonesian,Filipino": 0.42613636363636365, "Indonesian,Spanish": 0.4090909090909091, "Indonesian,Chinese": 0.19318181818181818, "Indonesian,Malay": 0.5568181818181818, "Indonesian,Vietnamese": 0.32386363636363635, "English,Filipino": 0.45454545454545453, "English,Spanish": 0.5511363636363636, "English,Chinese": 0.2840909090909091, "English,Malay": 0.375, "English,Vietnamese": 0.36363636363636365, "Filipino,Spanish": 0.4375, "Filipino,Chinese": 0.125, "Filipino,Malay": 0.4659090909090909, "Filipino,Vietnamese": 0.3125, "Spanish,Chinese": 0.1590909090909091, "Spanish,Malay": 0.4318181818181818, "Spanish,Vietnamese": 0.3522727272727273, "Chinese,Malay": 0.14204545454545456, "Chinese,Vietnamese": 0.4147727272727273, "Malay,Vietnamese": 0.3068181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.22727272727272727, "Indonesian,English,Spanish": 0.26704545454545453, "Indonesian,English,Chinese": 0.09659090909090909, "Indonesian,English,Malay": 0.25, "Indonesian,English,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish": 0.23863636363636365, "Indonesian,Filipino,Chinese": 0.056818181818181816, "Indonesian,Filipino,Malay": 0.2897727272727273, "Indonesian,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese": 0.0625, "Indonesian,Spanish,Malay": 0.29545454545454547, "Indonesian,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,Chinese,Malay": 0.09659090909090909, "Indonesian,Chinese,Vietnamese": 0.09090909090909091, "Indonesian,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish": 0.2897727272727273, "English,Filipino,Chinese": 0.08522727272727272, "English,Filipino,Malay": 0.22727272727272727, "English,Filipino,Vietnamese": 0.1875, "English,Spanish,Chinese": 0.11931818181818182, "English,Spanish,Malay": 0.25, "English,Spanish,Vietnamese": 0.21022727272727273, "English,Chinese,Malay": 0.08522727272727272, "English,Chinese,Vietnamese": 0.14772727272727273, "English,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese": 0.056818181818181816, "Filipino,Spanish,Malay": 0.24431818181818182, "Filipino,Spanish,Vietnamese": 0.19318181818181818, "Filipino,Chinese,Malay": 0.056818181818181816, "Filipino,Chinese,Vietnamese": 0.0625, "Filipino,Malay,Vietnamese": 0.17613636363636365, "Spanish,Chinese,Malay": 0.056818181818181816, "Spanish,Chinese,Vietnamese": 0.08522727272727272, "Spanish,Malay,Vietnamese": 0.17613636363636365, "Chinese,Malay,Vietnamese": 0.08522727272727272 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.17613636363636365, "Indonesian,English,Filipino,Chinese": 0.05113636363636364, "Indonesian,English,Filipino,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Vietnamese": 0.125, "Indonesian,English,Spanish,Chinese": 0.05113636363636364, "Indonesian,English,Spanish,Malay": 0.19318181818181818, "Indonesian,English,Spanish,Vietnamese": 0.125, "Indonesian,English,Chinese,Malay": 0.06818181818181818, "Indonesian,English,Chinese,Vietnamese": 0.05113636363636364, "Indonesian,English,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Chinese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Malay": 0.1875, "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay": 0.045454545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,Spanish,Chinese,Vietnamese": 0.03977272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.0625, "English,Filipino,Spanish,Chinese": 0.05113636363636364, "English,Filipino,Spanish,Malay": 0.16477272727272727, "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, "English,Filipino,Chinese,Malay": 0.045454545454545456, "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, "English,Filipino,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay": 0.056818181818181816, "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, "English,Spanish,Malay,Vietnamese": 0.10227272727272728, "English,Chinese,Malay,Vietnamese": 0.05113636363636364, "Filipino,Spanish,Chinese,Malay": 0.03409090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay": 0.03977272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.028409090909090908, "Indonesian,English,Filipino,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Chinese,Malay": 0.045454545454545456, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 } }, "AC3_2": 0.3215888798206052, "AC3_3": 0.20865746806576882, "AC3_4": 0.13533579752865357, "AC3_5": 0.09019505882135875, "AC3_6": 0.061076020734768734, "AC3_7": 0.042182752966308475 } }, "sg_eval": { "prompt_1": { "accuracy": 0.3592233009708738 }, "prompt_2": { "accuracy": 0.36893203883495146 }, "prompt_3": { "accuracy": 0.33980582524271846 }, "prompt_4": { "accuracy": 0.3300970873786408 }, "prompt_5": { "accuracy": 0.34951456310679613 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2571428571428571 }, "prompt_2": { "accuracy": 0.2571428571428571 }, "prompt_3": { "accuracy": 0.2571428571428571 }, "prompt_4": { "accuracy": 0.2571428571428571 }, "prompt_5": { "accuracy": 0.2571428571428571 } }, "us_eval": { "prompt_1": { "accuracy": 0.27102803738317754 }, "prompt_2": { "accuracy": 0.29906542056074764 }, "prompt_3": { "accuracy": 0.2616822429906542 }, "prompt_4": { "accuracy": 0.27102803738317754 }, "prompt_5": { "accuracy": 0.2616822429906542 } }, "ph_eval": { "prompt_1": { "accuracy": 0.28, "category_acc": { "brand": 0.4, "demographics": 0.8, "biology": 0.1, "history": 0.4666666666666667, "literature": 0.2, "politics": 0.2, "culture": 0.2, "film": 0.2, "law": 0.2, "geography": 0.2 } }, "prompt_2": { "accuracy": 0.28, "category_acc": { "brand": 0.4, "demographics": 0.6, "biology": 0.1, "history": 0.4666666666666667, "literature": 0.2, "politics": 0.3, "culture": 0.2, "film": 0.2, "law": 0.2, "geography": 0.2 } }, "prompt_3": { "accuracy": 0.23, "category_acc": { "brand": 0.3, "demographics": 0.4, "biology": 0.2, "history": 0.3333333333333333, "literature": 0.2, "politics": 0.2, "culture": 0.2, "film": 0.1, "law": 0.1, "geography": 0.3 } }, "prompt_4": { "accuracy": 0.31, "category_acc": { "brand": 0.4, "demographics": 0.8, "biology": 0.2, "history": 0.4666666666666667, "literature": 0.3, "politics": 0.2, "culture": 0.2, "film": 0.2, "law": 0.3, "geography": 0.2 } }, "prompt_5": { "accuracy": 0.28, "category_acc": { "brand": 0.3, "demographics": 0.8, "biology": 0.3, "history": 0.3333333333333333, "literature": 0.3, "politics": 0.2, "culture": 0.2, "film": 0.2, "law": 0.2, "geography": 0.2 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.1778634553271136 }, "prompt_2": { "bleu_score": 0.17587938617494803 }, "prompt_3": { "bleu_score": 0.18016948741331085 }, "prompt_4": { "bleu_score": 0.1755807406460382 }, "prompt_5": { "bleu_score": 0.15871208497539763 } }, "indommlu": { "prompt_1": { "accuracy": 0.22377995860871888, "category_acc": { "History": 0.15863453815261044, "Geography": 0.20612244897959184, "Lampungic": 0.25170068027210885, "Social science": 0.2621035058430718, "Balinese": 0.25902335456475584, "Makassarese": 0.23655913978494625, "Banjarese": 0.1736111111111111, "Chemistry": 0.1635036496350365, "Biology": 0.17159763313609466, "Science": 0.24045407636738905, "Christian religion": 0.22885572139303484, "Art": 0.22129783693843594, "Islam religion": 0.22048364153627312, "Hindu religion": 0.21333333333333335, "Madurese": 0.22033898305084745, "Sport": 0.19594594594594594, "Indonesian language": 0.24097135740971357, "Physics": 0.20202020202020202, "Minangkabau culture": 0.24623115577889448, "Dayak language": 0.28440366972477066, "Sociology": 0.1774193548387097, "Economy": 0.19672131147540983, "Sundanese": 0.24546240276577355, "Javanese": 0.2560483870967742, "Civic education": 0.23032904148783978 } }, "prompt_2": { "accuracy": 0.22077575272047534, "category_acc": { "History": 0.15060240963855423, "Geography": 0.2, "Lampungic": 0.2789115646258503, "Social science": 0.2637729549248748, "Balinese": 0.2611464968152866, "Makassarese": 0.21505376344086022, "Banjarese": 0.1736111111111111, "Chemistry": 0.1562043795620438, "Biology": 0.17159763313609466, "Science": 0.23839009287925697, "Christian religion": 0.22885572139303484, "Art": 0.22462562396006655, "Islam religion": 0.21906116642958748, "Hindu religion": 0.2, "Madurese": 0.2305084745762712, "Sport": 0.1891891891891892, "Indonesian language": 0.2387920298879203, "Physics": 0.19797979797979798, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.24770642201834864, "Sociology": 0.18346774193548387, "Economy": 0.17827868852459017, "Sundanese": 0.23768366464995677, "Javanese": 0.2560483870967742, "Civic education": 0.2246065808297568 } }, "prompt_3": { "accuracy": 0.22104279324387477, "category_acc": { "History": 0.15461847389558234, "Geography": 0.19795918367346937, "Lampungic": 0.24489795918367346, "Social science": 0.2587646076794658, "Balinese": 0.2611464968152866, "Makassarese": 0.23118279569892472, "Banjarese": 0.16666666666666666, "Chemistry": 0.1635036496350365, "Biology": 0.17514792899408285, "Science": 0.22910216718266255, "Christian religion": 0.23880597014925373, "Art": 0.2113144758735441, "Islam religion": 0.22617354196301565, "Hindu religion": 0.2, "Madurese": 0.20677966101694914, "Sport": 0.20945945945945946, "Indonesian language": 0.23692403486924035, "Physics": 0.19595959595959597, "Minangkabau culture": 0.23618090452261306, "Dayak language": 0.24770642201834864, "Sociology": 0.18346774193548387, "Economy": 0.19467213114754098, "Sundanese": 0.2428694900605013, "Javanese": 0.2620967741935484, "Civic education": 0.22746781115879827 } }, "prompt_4": { "accuracy": 0.22077575272047534, "category_acc": { "History": 0.15060240963855423, "Geography": 0.18775510204081633, "Lampungic": 0.23129251700680273, "Social science": 0.2587646076794658, "Balinese": 0.25690021231422505, "Makassarese": 0.22580645161290322, "Banjarese": 0.1736111111111111, "Chemistry": 0.1635036496350365, "Biology": 0.17514792899408285, "Science": 0.2301341589267286, "Christian religion": 0.23383084577114427, "Art": 0.2146422628951747, "Islam religion": 0.2233285917496444, "Hindu religion": 0.20666666666666667, "Madurese": 0.2135593220338983, "Sport": 0.1891891891891892, "Indonesian language": 0.2425280199252802, "Physics": 0.20202020202020202, "Minangkabau culture": 0.22613065326633167, "Dayak language": 0.24770642201834864, "Sociology": 0.18346774193548387, "Economy": 0.19262295081967212, "Sundanese": 0.24546240276577355, "Javanese": 0.2550403225806452, "Civic education": 0.21745350500715308 } }, "prompt_5": { "accuracy": 0.22731824554376126, "category_acc": { "History": 0.16265060240963855, "Geography": 0.19387755102040816, "Lampungic": 0.2789115646258503, "Social science": 0.2570951585976628, "Balinese": 0.2760084925690021, "Makassarese": 0.22580645161290322, "Banjarese": 0.18055555555555555, "Chemistry": 0.15766423357664233, "Biology": 0.178698224852071, "Science": 0.23529411764705882, "Christian religion": 0.23880597014925373, "Art": 0.22129783693843594, "Islam religion": 0.2233285917496444, "Hindu religion": 0.17333333333333334, "Madurese": 0.2576271186440678, "Sport": 0.21621621621621623, "Indonesian language": 0.24937733499377335, "Physics": 0.20202020202020202, "Minangkabau culture": 0.2613065326633166, "Dayak language": 0.29357798165137616, "Sociology": 0.19153225806451613, "Economy": 0.19057377049180327, "Sundanese": 0.2428694900605013, "Javanese": 0.2550403225806452, "Civic education": 0.24320457796852646 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.03374471826935659 }, "prompt_2": { "bleu_score": 0.03354584784772205 }, "prompt_3": { "bleu_score": 0.030709309536872887 }, "prompt_4": { "bleu_score": 0.033406005624105214 }, "prompt_5": { "bleu_score": 0.03408654681827874 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.023547771145113035 }, "prompt_2": { "bleu_score": 0.023013474143517277 }, "prompt_3": { "bleu_score": 0.020874466049663108 }, "prompt_4": { "bleu_score": 0.02411759361984998 }, "prompt_5": { "bleu_score": 0.02376678283625278 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.0061225167958979445 }, "prompt_2": { "bleu_score": 0.008007806300952039 }, "prompt_3": { "bleu_score": 0.008798554334606034 }, "prompt_4": { "bleu_score": 0.008702923563946885 }, "prompt_5": { "bleu_score": 0.00887049961254504 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.028766122560955086 }, "prompt_2": { "bleu_score": 0.03126042463241794 }, "prompt_3": { "bleu_score": 0.030200207989540616 }, "prompt_4": { "bleu_score": 0.03010036911944672 }, "prompt_5": { "bleu_score": 0.03042639392728816 } }, "mmlu": { "prompt_1": { "accuracy": 0.29638273045507585 }, "prompt_2": { "accuracy": 0.2928821470245041 }, "prompt_3": { "accuracy": 0.2870478413068845 }, "prompt_4": { "accuracy": 0.294049008168028 }, "prompt_5": { "accuracy": 0.2847141190198366 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.2889524490525563, "category_acc": { "high_school_european_history": 0.3231707317073171, "business_ethics": 0.32323232323232326, "clinical_knowledge": 0.30303030303030304, "medical_genetics": 0.1919191919191919, "high_school_us_history": 0.3399014778325123, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.3220338983050847, "virology": 0.26666666666666666, "high_school_microeconomics": 0.3291139240506329, "econometrics": 0.24778761061946902, "college_computer_science": 0.25252525252525254, "high_school_biology": 0.2977346278317152, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.26334519572953735, "philosophy": 0.3, "professional_medicine": 0.36531365313653136, "nutrition": 0.3442622950819672, "global_facts": 0.21212121212121213, "machine_learning": 0.17117117117117117, "security_studies": 0.22950819672131148, "public_relations": 0.3302752293577982, "professional_psychology": 0.2684124386252046, "prehistory": 0.28792569659442724, "anatomy": 0.2462686567164179, "human_sexuality": 0.3384615384615385, "college_medicine": 0.26744186046511625, "high_school_government_and_politics": 0.328125, "college_chemistry": 0.29292929292929293, "logical_fallacies": 0.345679012345679, "high_school_geography": 0.27411167512690354, "elementary_mathematics": 0.26790450928381965, "human_aging": 0.25225225225225223, "college_mathematics": 0.23232323232323232, "high_school_psychology": 0.3180147058823529, "formal_logic": 0.32, "high_school_statistics": 0.25116279069767444, "international_law": 0.2916666666666667, "high_school_mathematics": 0.2527881040892193, "high_school_computer_science": 0.26262626262626265, "conceptual_physics": 0.29914529914529914, "miscellaneous": 0.27365728900255754, "high_school_chemistry": 0.23267326732673269, "marketing": 0.41630901287553645, "professional_law": 0.30071754729288974, "management": 0.3627450980392157, "college_physics": 0.19801980198019803, "jurisprudence": 0.29906542056074764, "world_religions": 0.23529411764705882, "sociology": 0.325, "us_foreign_policy": 0.35353535353535354, "high_school_macroeconomics": 0.3033419023136247, "computer_security": 0.21212121212121213, "moral_scenarios": 0.24608501118568232, "moral_disputes": 0.2898550724637681, "electrical_engineering": 0.2986111111111111, "astronomy": 0.3443708609271523, "college_biology": 0.27972027972027974 } }, "prompt_2": { "accuracy": 0.2925277082588488, "category_acc": { "high_school_european_history": 0.3353658536585366, "business_ethics": 0.31313131313131315, "clinical_knowledge": 0.32196969696969696, "medical_genetics": 0.1919191919191919, "high_school_us_history": 0.3399014778325123, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.3305084745762712, "virology": 0.2606060606060606, "high_school_microeconomics": 0.2742616033755274, "econometrics": 0.22123893805309736, "college_computer_science": 0.2828282828282828, "high_school_biology": 0.2783171521035599, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.25622775800711745, "philosophy": 0.3225806451612903, "professional_medicine": 0.33579335793357934, "nutrition": 0.32131147540983607, "global_facts": 0.29292929292929293, "machine_learning": 0.27927927927927926, "security_studies": 0.22950819672131148, "public_relations": 0.3119266055045872, "professional_psychology": 0.2765957446808511, "prehistory": 0.30340557275541796, "anatomy": 0.22388059701492538, "human_sexuality": 0.3076923076923077, "college_medicine": 0.3023255813953488, "high_school_government_and_politics": 0.3072916666666667, "college_chemistry": 0.24242424242424243, "logical_fallacies": 0.3333333333333333, "high_school_geography": 0.28426395939086296, "elementary_mathematics": 0.23342175066312998, "human_aging": 0.3108108108108108, "college_mathematics": 0.29292929292929293, "high_school_psychology": 0.3235294117647059, "formal_logic": 0.288, "high_school_statistics": 0.24651162790697675, "international_law": 0.3333333333333333, "high_school_mathematics": 0.2788104089219331, "high_school_computer_science": 0.32323232323232326, "conceptual_physics": 0.2948717948717949, "miscellaneous": 0.30818414322250637, "high_school_chemistry": 0.22772277227722773, "marketing": 0.44206008583690987, "professional_law": 0.29549902152641877, "management": 0.3235294117647059, "college_physics": 0.2079207920792079, "jurisprudence": 0.32710280373831774, "world_religions": 0.2529411764705882, "sociology": 0.355, "us_foreign_policy": 0.31313131313131315, "high_school_macroeconomics": 0.29562982005141386, "computer_security": 0.31313131313131315, "moral_scenarios": 0.24720357941834453, "moral_disputes": 0.3217391304347826, "electrical_engineering": 0.3194444444444444, "astronomy": 0.2847682119205298, "college_biology": 0.26573426573426573 } }, "prompt_3": { "accuracy": 0.285305684662138, "category_acc": { "high_school_european_history": 0.3231707317073171, "business_ethics": 0.2727272727272727, "clinical_knowledge": 0.29924242424242425, "medical_genetics": 0.16161616161616163, "high_school_us_history": 0.33497536945812806, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.3347457627118644, "virology": 0.3090909090909091, "high_school_microeconomics": 0.3037974683544304, "econometrics": 0.23008849557522124, "college_computer_science": 0.24242424242424243, "high_school_biology": 0.3042071197411003, "abstract_algebra": 0.21212121212121213, "professional_accounting": 0.23487544483985764, "philosophy": 0.27741935483870966, "professional_medicine": 0.35793357933579334, "nutrition": 0.3016393442622951, "global_facts": 0.2222222222222222, "machine_learning": 0.27927927927927926, "security_studies": 0.2540983606557377, "public_relations": 0.3119266055045872, "professional_psychology": 0.2602291325695581, "prehistory": 0.2848297213622291, "anatomy": 0.21641791044776118, "human_sexuality": 0.27692307692307694, "college_medicine": 0.3023255813953488, "high_school_government_and_politics": 0.3541666666666667, "college_chemistry": 0.26262626262626265, "logical_fallacies": 0.3333333333333333, "high_school_geography": 0.28426395939086296, "elementary_mathematics": 0.22015915119363394, "human_aging": 0.26576576576576577, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.3382352941176471, "formal_logic": 0.272, "high_school_statistics": 0.20465116279069767, "international_law": 0.25, "high_school_mathematics": 0.22304832713754646, "high_school_computer_science": 0.2828282828282828, "conceptual_physics": 0.29914529914529914, "miscellaneous": 0.2749360613810742, "high_school_chemistry": 0.21287128712871287, "marketing": 0.43776824034334766, "professional_law": 0.28701891715590344, "management": 0.35294117647058826, "college_physics": 0.19801980198019803, "jurisprudence": 0.32710280373831774, "world_religions": 0.2411764705882353, "sociology": 0.355, "us_foreign_policy": 0.30303030303030304, "high_school_macroeconomics": 0.30848329048843187, "computer_security": 0.24242424242424243, "moral_scenarios": 0.2505592841163311, "moral_disputes": 0.32463768115942027, "electrical_engineering": 0.3263888888888889, "astronomy": 0.36423841059602646, "college_biology": 0.27972027972027974 } }, "prompt_4": { "accuracy": 0.2910260993922059, "category_acc": { "high_school_european_history": 0.34146341463414637, "business_ethics": 0.29292929292929293, "clinical_knowledge": 0.2765151515151515, "medical_genetics": 0.1919191919191919, "high_school_us_history": 0.3694581280788177, "high_school_physics": 0.26, "high_school_world_history": 0.3050847457627119, "virology": 0.26666666666666666, "high_school_microeconomics": 0.28270042194092826, "econometrics": 0.26548672566371684, "college_computer_science": 0.2727272727272727, "high_school_biology": 0.2977346278317152, "abstract_algebra": 0.24242424242424243, "professional_accounting": 0.24555160142348753, "philosophy": 0.2903225806451613, "professional_medicine": 0.35793357933579334, "nutrition": 0.32131147540983607, "global_facts": 0.20202020202020202, "machine_learning": 0.2882882882882883, "security_studies": 0.29098360655737704, "public_relations": 0.3211009174311927, "professional_psychology": 0.28805237315875615, "prehistory": 0.29102167182662536, "anatomy": 0.2835820895522388, "human_sexuality": 0.3230769230769231, "college_medicine": 0.27906976744186046, "high_school_government_and_politics": 0.3072916666666667, "college_chemistry": 0.24242424242424243, "logical_fallacies": 0.35802469135802467, "high_school_geography": 0.28426395939086296, "elementary_mathematics": 0.27055702917771884, "human_aging": 0.2702702702702703, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.3161764705882353, "formal_logic": 0.24, "high_school_statistics": 0.26976744186046514, "international_law": 0.275, "high_school_mathematics": 0.22676579925650558, "high_school_computer_science": 0.29292929292929293, "conceptual_physics": 0.29914529914529914, "miscellaneous": 0.2979539641943734, "high_school_chemistry": 0.22772277227722773, "marketing": 0.39914163090128757, "professional_law": 0.29158512720156554, "management": 0.3431372549019608, "college_physics": 0.2079207920792079, "jurisprudence": 0.34579439252336447, "world_religions": 0.25882352941176473, "sociology": 0.345, "us_foreign_policy": 0.30303030303030304, "high_school_macroeconomics": 0.29562982005141386, "computer_security": 0.29292929292929293, "moral_scenarios": 0.24608501118568232, "moral_disputes": 0.3159420289855073, "electrical_engineering": 0.3333333333333333, "astronomy": 0.3576158940397351, "college_biology": 0.3076923076923077 } }, "prompt_5": { "accuracy": 0.29789059706828747, "category_acc": { "high_school_european_history": 0.3902439024390244, "business_ethics": 0.2828282828282828, "clinical_knowledge": 0.3143939393939394, "medical_genetics": 0.25252525252525254, "high_school_us_history": 0.35960591133004927, "high_school_physics": 0.22666666666666666, "high_school_world_history": 0.3093220338983051, "virology": 0.3090909090909091, "high_school_microeconomics": 0.2911392405063291, "econometrics": 0.22123893805309736, "college_computer_science": 0.25252525252525254, "high_school_biology": 0.32038834951456313, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.29537366548042704, "philosophy": 0.3193548387096774, "professional_medicine": 0.3025830258302583, "nutrition": 0.33114754098360655, "global_facts": 0.3434343434343434, "machine_learning": 0.3333333333333333, "security_studies": 0.2459016393442623, "public_relations": 0.3577981651376147, "professional_psychology": 0.2864157119476268, "prehistory": 0.32507739938080493, "anatomy": 0.26119402985074625, "human_sexuality": 0.2846153846153846, "college_medicine": 0.29069767441860467, "high_school_government_and_politics": 0.2916666666666667, "college_chemistry": 0.21212121212121213, "logical_fallacies": 0.36419753086419754, "high_school_geography": 0.3147208121827411, "elementary_mathematics": 0.22811671087533156, "human_aging": 0.2927927927927928, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.32536764705882354, "formal_logic": 0.288, "high_school_statistics": 0.21395348837209302, "international_law": 0.38333333333333336, "high_school_mathematics": 0.2527881040892193, "high_school_computer_science": 0.29292929292929293, "conceptual_physics": 0.33760683760683763, "miscellaneous": 0.3171355498721228, "high_school_chemistry": 0.26732673267326734, "marketing": 0.44635193133047213, "professional_law": 0.2922374429223744, "management": 0.23529411764705882, "college_physics": 0.1782178217821782, "jurisprudence": 0.35514018691588783, "world_religions": 0.27647058823529413, "sociology": 0.36, "us_foreign_policy": 0.36363636363636365, "high_school_macroeconomics": 0.2930591259640103, "computer_security": 0.31313131313131315, "moral_scenarios": 0.24720357941834453, "moral_disputes": 0.3130434782608696, "electrical_engineering": 0.3333333333333333, "astronomy": 0.33112582781456956, "college_biology": 0.2727272727272727 } } }, "c_eval": { "prompt_1": { "accuracy": 0.24591381872213966 }, "prompt_2": { "accuracy": 0.24591381872213966 }, "prompt_3": { "accuracy": 0.24591381872213966 }, "prompt_4": { "accuracy": 0.24294205052005943 }, "prompt_5": { "accuracy": 0.24591381872213966 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.25840597758405975, "category_acc": { "computer_network": 0.25, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.23809523809523808, "college_physics": 0.20833333333333334, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.2, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.37037037037037035, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.23076923076923078, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.375, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3333333333333333, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.2777777777777778, "physician": 0.2962962962962963 } }, "prompt_2": { "accuracy": 0.2577833125778331, "category_acc": { "computer_network": 0.25, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.21428571428571427, "college_physics": 0.20833333333333334, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.2, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.37037037037037035, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.23076923076923078, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.375, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3333333333333333, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.2777777777777778, "physician": 0.2962962962962963 } }, "prompt_3": { "accuracy": 0.25840597758405975, "category_acc": { "computer_network": 0.25, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.23809523809523808, "college_physics": 0.20833333333333334, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.2, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.37037037037037035, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.23076923076923078, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.375, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3333333333333333, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.2777777777777778, "physician": 0.2962962962962963 } }, "prompt_4": { "accuracy": 0.25840597758405975, "category_acc": { "computer_network": 0.25, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.21428571428571427, "college_physics": 0.20833333333333334, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.25, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.2, "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.37037037037037035, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.23076923076923078, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.375, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3333333333333333, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.2777777777777778, "physician": 0.2962962962962963 } }, "prompt_5": { "accuracy": 0.25840597758405975, "category_acc": { "computer_network": 0.25, "operating_system": 0.4166666666666667, "computer_architecture": 0.23076923076923078, "college_programming": 0.23809523809523808, "college_physics": 0.20833333333333334, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.375, "high_school_biology": 0.25, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.2, "business_administration": 0.13157894736842105, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.37037037037037035, "law": 0.3448275862068966, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.08, "middle_school_history": 0.3333333333333333, "civil_servant": 0.23076923076923078, "sports_science": 0.2916666666666667, "plant_protection": 0.14814814814814814, "basic_medicine": 0.375, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3333333333333333, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.2777777777777778, "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.2078853046594982 }, "prompt_2": { "accuracy": 0.2114695340501792 }, "prompt_3": { "accuracy": 0.2078853046594982 }, "prompt_4": { "accuracy": 0.2114695340501792 }, "prompt_5": { "accuracy": 0.2114695340501792 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2487480573303402, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.25675675675675674, "ancient_chinese": 0.23780487804878048, "arts": 0.25, "astronomy": 0.2727272727272727, "business_ethics": 0.2631578947368421, "chinese_civil_service_exam": 0.26875, "chinese_driving_rule": 0.21374045801526717, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.2476780185758514, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.2569832402234637, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.2830188679245283, "college_education": 0.14953271028037382, "college_engineering_hydrology": 0.27358490566037735, "college_law": 0.32407407407407407, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.20754716981132076, "college_medicine": 0.2271062271062271, "computer_science": 0.24019607843137256, "computer_security": 0.24561403508771928, "conceptual_physics": 0.23129251700680273, "construction_project_management": 0.2446043165467626, "economics": 0.2578616352201258, "education": 0.25153374233128833, "electrical_engineering": 0.25, "elementary_chinese": 0.20634920634920634, "elementary_commonsense": 0.2727272727272727, "elementary_information_and_technology": 0.226890756302521, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.2518518518518518, "food_science": 0.26573426573426573, "genetics": 0.2727272727272727, "global_facts": 0.26174496644295303, "high_school_biology": 0.2781065088757396, "high_school_chemistry": 0.19696969696969696, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.24475524475524477, "human_sexuality": 0.2698412698412698, "international_law": 0.25405405405405407, "journalism": 0.2558139534883721, "jurisprudence": 0.26034063260340634, "legal_and_moral_basis": 0.2336448598130841, "logical": 0.23577235772357724, "machine_learning": 0.2540983606557377, "management": 0.24761904761904763, "marketing": 0.2388888888888889, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25, "nutrition": 0.27586206896551724, "philosophy": 0.24761904761904763, "professional_accounting": 0.25142857142857145, "professional_law": 0.2559241706161137, "professional_medicine": 0.25, "professional_psychology": 0.2543103448275862, "public_relations": 0.23563218390804597, "security_study": 0.2222222222222222, "sociology": 0.25663716814159293, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.23076923076923078, "world_history": 0.2546583850931677, "world_religions": 0.26875 } }, "prompt_2": { "accuracy": 0.2490070799516491, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.25675675675675674, "ancient_chinese": 0.23780487804878048, "arts": 0.25, "astronomy": 0.2727272727272727, "business_ethics": 0.2631578947368421, "chinese_civil_service_exam": 0.26875, "chinese_driving_rule": 0.21374045801526717, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.2476780185758514, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.2569832402234637, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.29245283018867924, "college_education": 0.14953271028037382, "college_engineering_hydrology": 0.27358490566037735, "college_law": 0.32407407407407407, "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.20754716981132076, "college_medicine": 0.2271062271062271, "computer_science": 0.23529411764705882, "computer_security": 0.25146198830409355, "conceptual_physics": 0.23129251700680273, "construction_project_management": 0.2446043165467626, "economics": 0.2578616352201258, "education": 0.25153374233128833, "electrical_engineering": 0.2558139534883721, "elementary_chinese": 0.20634920634920634, "elementary_commonsense": 0.2727272727272727, "elementary_information_and_technology": 0.2184873949579832, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.2518518518518518, "food_science": 0.2727272727272727, "genetics": 0.2727272727272727, "global_facts": 0.26174496644295303, "high_school_biology": 0.28402366863905326, "high_school_chemistry": 0.19696969696969696, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.24475524475524477, "human_sexuality": 0.2698412698412698, "international_law": 0.25405405405405407, "journalism": 0.2558139534883721, "jurisprudence": 0.26034063260340634, "legal_and_moral_basis": 0.2336448598130841, "logical": 0.23577235772357724, "machine_learning": 0.2540983606557377, "management": 0.24761904761904763, "marketing": 0.2388888888888889, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25, "nutrition": 0.27586206896551724, "philosophy": 0.24761904761904763, "professional_accounting": 0.25142857142857145, "professional_law": 0.2559241706161137, "professional_medicine": 0.25, "professional_psychology": 0.2543103448275862, "public_relations": 0.23563218390804597, "security_study": 0.2222222222222222, "sociology": 0.25663716814159293, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.23076923076923078, "world_history": 0.2546583850931677, "world_religions": 0.26875 } }, "prompt_3": { "accuracy": 0.2487480573303402, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.25675675675675674, "ancient_chinese": 0.23780487804878048, "arts": 0.25, "astronomy": 0.2727272727272727, "business_ethics": 0.2631578947368421, "chinese_civil_service_exam": 0.26875, "chinese_driving_rule": 0.21374045801526717, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.25386996904024767, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.2569832402234637, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.2830188679245283, "college_education": 0.14953271028037382, "college_engineering_hydrology": 0.27358490566037735, "college_law": 0.32407407407407407, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.20754716981132076, "college_medicine": 0.2271062271062271, "computer_science": 0.23529411764705882, "computer_security": 0.25146198830409355, "conceptual_physics": 0.23129251700680273, "construction_project_management": 0.2446043165467626, "economics": 0.2578616352201258, "education": 0.25153374233128833, "electrical_engineering": 0.2558139534883721, "elementary_chinese": 0.20634920634920634, "elementary_commonsense": 0.2727272727272727, "elementary_information_and_technology": 0.22268907563025211, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.2518518518518518, "food_science": 0.26573426573426573, "genetics": 0.2727272727272727, "global_facts": 0.26174496644295303, "high_school_biology": 0.28402366863905326, "high_school_chemistry": 0.18181818181818182, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.24475524475524477, "human_sexuality": 0.2698412698412698, "international_law": 0.25405405405405407, "journalism": 0.2558139534883721, "jurisprudence": 0.26034063260340634, "legal_and_moral_basis": 0.2336448598130841, "logical": 0.23577235772357724, "machine_learning": 0.2540983606557377, "management": 0.24761904761904763, "marketing": 0.2388888888888889, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25, "nutrition": 0.27586206896551724, "philosophy": 0.24761904761904763, "professional_accounting": 0.25142857142857145, "professional_law": 0.2559241706161137, "professional_medicine": 0.25, "professional_psychology": 0.2543103448275862, "public_relations": 0.23563218390804597, "security_study": 0.2222222222222222, "sociology": 0.25663716814159293, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.23076923076923078, "world_history": 0.2546583850931677, "world_religions": 0.26875 } }, "prompt_4": { "accuracy": 0.2479709894664134, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.25675675675675674, "ancient_chinese": 0.23780487804878048, "arts": 0.25, "astronomy": 0.26666666666666666, "business_ethics": 0.2679425837320574, "chinese_civil_service_exam": 0.275, "chinese_driving_rule": 0.22137404580152673, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.2476780185758514, "chinese_literature": 0.2549019607843137, "chinese_teacher_qualification": 0.24581005586592178, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.29245283018867924, "college_education": 0.14953271028037382, "college_engineering_hydrology": 0.27358490566037735, "college_law": 0.32407407407407407, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.20754716981132076, "college_medicine": 0.2271062271062271, "computer_science": 0.23529411764705882, "computer_security": 0.25146198830409355, "conceptual_physics": 0.23129251700680273, "construction_project_management": 0.2446043165467626, "economics": 0.2578616352201258, "education": 0.25153374233128833, "electrical_engineering": 0.23255813953488372, "elementary_chinese": 0.20238095238095238, "elementary_commonsense": 0.2727272727272727, "elementary_information_and_technology": 0.2184873949579832, "elementary_mathematics": 0.25217391304347825, "ethnology": 0.24444444444444444, "food_science": 0.25874125874125875, "genetics": 0.26704545454545453, "global_facts": 0.26174496644295303, "high_school_biology": 0.2603550295857988, "high_school_chemistry": 0.20454545454545456, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.25, "high_school_physics": 0.2727272727272727, "high_school_politics": 0.23776223776223776, "human_sexuality": 0.2698412698412698, "international_law": 0.2594594594594595, "journalism": 0.2558139534883721, "jurisprudence": 0.26034063260340634, "legal_and_moral_basis": 0.22897196261682243, "logical": 0.23577235772357724, "machine_learning": 0.2459016393442623, "management": 0.2523809523809524, "marketing": 0.2388888888888889, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25, "nutrition": 0.2827586206896552, "philosophy": 0.2571428571428571, "professional_accounting": 0.24571428571428572, "professional_law": 0.24644549763033174, "professional_medicine": 0.24202127659574468, "professional_psychology": 0.25, "public_relations": 0.2413793103448276, "security_study": 0.22962962962962963, "sociology": 0.25663716814159293, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.2485207100591716, "world_history": 0.2546583850931677, "world_religions": 0.26875 } }, "prompt_5": { "accuracy": 0.24831635296149196, "category_acc": { "agronomy": 0.23668639053254437, "anatomy": 0.25675675675675674, "ancient_chinese": 0.23780487804878048, "arts": 0.25, "astronomy": 0.2727272727272727, "business_ethics": 0.2631578947368421, "chinese_civil_service_exam": 0.26875, "chinese_driving_rule": 0.21374045801526717, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.22429906542056074, "chinese_history": 0.2476780185758514, "chinese_literature": 0.25, "chinese_teacher_qualification": 0.2569832402234637, "clinical_knowledge": 0.24472573839662448, "college_actuarial_science": 0.2830188679245283, "college_education": 0.14953271028037382, "college_engineering_hydrology": 0.27358490566037735, "college_law": 0.32407407407407407, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.20754716981132076, "college_medicine": 0.2271062271062271, "computer_science": 0.23529411764705882, "computer_security": 0.24561403508771928, "conceptual_physics": 0.23129251700680273, "construction_project_management": 0.2446043165467626, "economics": 0.2578616352201258, "education": 0.25153374233128833, "electrical_engineering": 0.25, "elementary_chinese": 0.20634920634920634, "elementary_commonsense": 0.2727272727272727, "elementary_information_and_technology": 0.2184873949579832, "elementary_mathematics": 0.24347826086956523, "ethnology": 0.2518518518518518, "food_science": 0.26573426573426573, "genetics": 0.26704545454545453, "global_facts": 0.26174496644295303, "high_school_biology": 0.2781065088757396, "high_school_chemistry": 0.19696969696969696, "high_school_geography": 0.23728813559322035, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.24475524475524477, "human_sexuality": 0.2698412698412698, "international_law": 0.25405405405405407, "journalism": 0.2558139534883721, "jurisprudence": 0.26034063260340634, "legal_and_moral_basis": 0.2336448598130841, "logical": 0.23577235772357724, "machine_learning": 0.2540983606557377, "management": 0.24761904761904763, "marketing": 0.2388888888888889, "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25, "nutrition": 0.27586206896551724, "philosophy": 0.24761904761904763, "professional_accounting": 0.25142857142857145, "professional_law": 0.2559241706161137, "professional_medicine": 0.25, "professional_psychology": 0.2543103448275862, "public_relations": 0.23563218390804597, "security_study": 0.2222222222222222, "sociology": 0.25663716814159293, "sports_science": 0.2727272727272727, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.23076923076923078, "world_history": 0.2546583850931677, "world_religions": 0.26875 } } }, "zbench": { "prompt_1": { "accuracy": 0.3939393939393939 }, "prompt_2": { "accuracy": 0.3939393939393939 }, "prompt_3": { "accuracy": 0.3939393939393939 }, "prompt_4": { "accuracy": 0.42424242424242425 }, "prompt_5": { "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.19090909090909092 }, "prompt_2": { "accuracy": 0.20909090909090908 }, "prompt_3": { "accuracy": 0.20454545454545456 }, "prompt_4": { "accuracy": 0.24772727272727274 }, "prompt_5": { "accuracy": 0.2409090909090909 } }, "ocnli": { "prompt_1": { "accuracy": 0.34508474576271186 }, "prompt_2": { "accuracy": 0.32440677966101694 }, "prompt_3": { "accuracy": 0.3233898305084746 }, "prompt_4": { "accuracy": 0.3352542372881356 }, "prompt_5": { "accuracy": 0.3352542372881356 } }, "c3": { "prompt_1": { "accuracy": 0.2830964846671653 }, "prompt_2": { "accuracy": 0.2793567688855647 }, "prompt_3": { "accuracy": 0.2793567688855647 }, "prompt_4": { "accuracy": 0.28272251308900526 }, "prompt_5": { "accuracy": 0.2793567688855647 } }, "dream": { "prompt_1": { "accuracy": 0.5100440960313571 }, "prompt_2": { "accuracy": 0.5213130818226359 }, "prompt_3": { "accuracy": 0.512983831455169 }, "prompt_4": { "accuracy": 0.5154336109750123 }, "prompt_5": { "accuracy": 0.5071043606075453 } }, "samsum": { "prompt_1": { "rouge1": 0.43481455633608285, "rouge2": 0.19020306710468923, "rougeL": 0.3555281762061519, "avg_rouge": 0.326848599882308 }, "prompt_2": { "rouge1": 0.43319141489410384, "rouge2": 0.18633011055371998, "rougeL": 0.3530126019631636, "avg_rouge": 0.3241780424703291 }, "prompt_3": { "rouge1": 0.4303838979441285, "rouge2": 0.18417594383860397, "rougeL": 0.34948463192329804, "avg_rouge": 0.32134815790201016 }, "prompt_4": { "rouge1": 0.43070227421856316, "rouge2": 0.18614737183994354, "rougeL": 0.3529860498921421, "avg_rouge": 0.3232785653168829 }, "prompt_5": { "rouge1": 0.4308048074328594, "rouge2": 0.18586059514617964, "rougeL": 0.3528154914643337, "avg_rouge": 0.3231602980144576 } }, "dialogsum": { "prompt_1": { "rouge1": 0.23467138626961317, "rouge2": 0.06744069122334906, "rougeL": 0.1779908580361024, "avg_rouge": 0.16003431184302153 }, "prompt_2": { "rouge1": 0.2483010108284747, "rouge2": 0.075125300962018, "rougeL": 0.18996631825453364, "avg_rouge": 0.17113087668167545 }, "prompt_3": { "rouge1": 0.23828052499310123, "rouge2": 0.07068930036983678, "rougeL": 0.18168303338974037, "avg_rouge": 0.16355095291755947 }, "prompt_4": { "rouge1": 0.2552198528902313, "rouge2": 0.07694264335968432, "rougeL": 0.19993614328948328, "avg_rouge": 0.17736621317979964 }, "prompt_5": { "rouge1": 0.22558914523023577, "rouge2": 0.0636998106706898, "rougeL": 0.1683448949296202, "avg_rouge": 0.15254461694351526 } }, "sst2": { "prompt_1": { "accuracy": 0.856651376146789 }, "prompt_2": { "accuracy": 0.8486238532110092 }, "prompt_3": { "accuracy": 0.8451834862385321 }, "prompt_4": { "accuracy": 0.8577981651376146 }, "prompt_5": { "accuracy": 0.8692660550458715 } }, "cola": { "prompt_1": { "accuracy": 0.6720997123681688 }, "prompt_2": { "accuracy": 0.6663470757430489 }, "prompt_3": { "accuracy": 0.6816874400767018 }, "prompt_4": { "accuracy": 0.663470757430489 }, "prompt_5": { "accuracy": 0.6519654841802492 } }, "qqp": { "prompt_1": { "accuracy": 0.474 }, "prompt_2": { "accuracy": 0.4865 }, "prompt_3": { "accuracy": 0.4485 }, "prompt_4": { "accuracy": 0.4995 }, "prompt_5": { "accuracy": 0.4935 } }, "mnli": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.2 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } }, "qnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.4 } }, "wnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.7 } }, "rte": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } }, "mrpc": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.5 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "mt0_xxl": { "model_size": "13B", "model_link": "https://huggingface.co/bigscience/mt0-xxl", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9678571428571429, "language_acc": { "Spanish": 0.9705882352941176, "English": 0.9722689075630252, "Chinese": 0.957983193277311, "Vietnamese": 0.9705882352941176 }, "consistency_score_2": 0.9642857142857143, "consistency_score_3": 0.9474789915966386, "consistency_score_4": 0.9352941176470588, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9773109243697479, "Spanish,Chinese": 0.957983193277311, "Spanish,Vietnamese": 0.9680672268907563, "English,Chinese": 0.9596638655462185, "English,Vietnamese": 0.9680672268907563, "Chinese,Vietnamese": 0.9546218487394958 }, "3_combine": { "Spanish,English,Chinese": 0.9487394957983193, "Spanish,English,Vietnamese": 0.957983193277311, "Spanish,Chinese,Vietnamese": 0.9411764705882353, "English,Chinese,Vietnamese": 0.9420168067226891 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9352941176470588 } }, "AC3_2": 0.9660681277556511, "AC3_3": 0.9575596608660338, "AC3_4": 0.9512970526048331 }, "prompt_2": { "overall_acc": 0.969327731092437, "language_acc": { "Spanish": 0.9722689075630252, "English": 0.9747899159663865, "Chinese": 0.957983193277311, "Vietnamese": 0.9722689075630252 }, "consistency_score_2": 0.9656862745098039, "consistency_score_3": 0.9493697478991596, "consistency_score_4": 0.9369747899159664, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9781512605042016, "Spanish,Chinese": 0.9571428571428572, "Spanish,Vietnamese": 0.9722689075630252, "English,Chinese": 0.9588235294117647, "English,Vietnamese": 0.9714285714285714, "Chinese,Vietnamese": 0.9563025210084034 }, "3_combine": { "Spanish,English,Chinese": 0.9478991596638655, "Spanish,English,Vietnamese": 0.9621848739495799, "Spanish,Chinese,Vietnamese": 0.9428571428571428, "English,Chinese,Vietnamese": 0.9445378151260504 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9369747899159664 } }, "AC3_2": 0.9675035763660907, "AC3_3": 0.9592449395770114, "AC3_4": 0.9528767203979467 }, "prompt_3": { "overall_acc": 0.9689075630252101, "language_acc": { "Spanish": 0.9722689075630252, "English": 0.9756302521008403, "Chinese": 0.9554621848739496, "Vietnamese": 0.9722689075630252 }, "consistency_score_2": 0.965546218487395, "consistency_score_3": 0.9493697478991596, "consistency_score_4": 0.9369747899159664, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9815126050420168, "Spanish,Chinese": 0.9537815126050421, "Spanish,Vietnamese": 0.973109243697479, "English,Chinese": 0.9554621848739496, "English,Vietnamese": 0.9739495798319328, "Chinese,Vietnamese": 0.9554621848739496 }, "3_combine": { "Spanish,English,Chinese": 0.946218487394958, "Spanish,English,Vietnamese": 0.965546218487395, "Spanish,Chinese,Vietnamese": 0.9420168067226891, "English,Chinese,Vietnamese": 0.9436974789915966 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9369747899159664 } }, "AC3_2": 0.9672239703374601, "AC3_3": 0.9590391582702165, "AC3_4": 0.9526736620074296 }, "prompt_4": { "overall_acc": 0.9686974789915966, "language_acc": { "Spanish": 0.9722689075630252, "English": 0.9747899159663865, "Chinese": 0.9563025210084034, "Vietnamese": 0.9714285714285714 }, "consistency_score_2": 0.9656862745098039, "consistency_score_3": 0.9495798319327731, "consistency_score_4": 0.9369747899159664, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9798319327731092, "Spanish,Chinese": 0.9563025210084034, "Spanish,Vietnamese": 0.9722689075630252, "English,Chinese": 0.9596638655462185, "English,Vietnamese": 0.9705882352941176, "Chinese,Vietnamese": 0.9554621848739496 }, "3_combine": { "Spanish,English,Chinese": 0.9487394957983193, "Spanish,English,Vietnamese": 0.9630252100840336, "Spanish,Chinese,Vietnamese": 0.9428571428571428, "English,Chinese,Vietnamese": 0.9436974789915966 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9369747899159664 } }, "AC3_2": 0.9671895329691594, "AC3_3": 0.959043391701624, "AC3_4": 0.9525720992340004 }, "prompt_5": { "overall_acc": 0.9697478991596639, "language_acc": { "Spanish": 0.9714285714285714, "English": 0.9747899159663865, "Chinese": 0.9588235294117647, "Vietnamese": 0.9739495798319328 }, "consistency_score_2": 0.9666666666666667, "consistency_score_3": 0.9508403361344537, "consistency_score_4": 0.938655462184874, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9798319327731092, "Spanish,Chinese": 0.9571428571428572, "Spanish,Vietnamese": 0.9705882352941176, "English,Chinese": 0.9588235294117647, "English,Vietnamese": 0.9739495798319328, "Chinese,Vietnamese": 0.9596638655462185 }, "3_combine": { "Spanish,English,Chinese": 0.9487394957983193, "Spanish,English,Vietnamese": 0.9630252100840336, "Spanish,Chinese,Vietnamese": 0.9445378151260504, "English,Chinese,Vietnamese": 0.9470588235294117 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.938655462184874 } }, "AC3_2": 0.9682048314269277, "AC3_3": 0.960201048210044, "AC3_4": 0.9539483956887579 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.45142857142857146, "language_acc": { "Filipino": 0.4533333333333333, "Vietnamese": 0.43333333333333335, "Chinese": 0.4533333333333333, "Spanish": 0.42, "Malay": 0.48, "Indonesian": 0.46, "English": 0.46 }, "consistency_score_2": 0.6977777777777778, "consistency_score_3": 0.570857142857143, "consistency_score_4": 0.49542857142857144, "consistency_score_5": 0.44317460317460317, "consistency_score_6": 0.40380952380952373, "consistency_score_7": 0.37333333333333335, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.7066666666666667, "Filipino,Chinese": 0.5933333333333334, "Filipino,Spanish": 0.7266666666666667, "Filipino,Malay": 0.74, "Filipino,Indonesian": 0.7, "Filipino,English": 0.7733333333333333, "Vietnamese,Chinese": 0.6266666666666667, "Vietnamese,Spanish": 0.6733333333333333, "Vietnamese,Malay": 0.7066666666666667, "Vietnamese,Indonesian": 0.7, "Vietnamese,English": 0.7133333333333334, "Chinese,Spanish": 0.6333333333333333, "Chinese,Malay": 0.5733333333333334, "Chinese,Indonesian": 0.6533333333333333, "Chinese,English": 0.6466666666666666, "Spanish,Malay": 0.7266666666666667, "Spanish,Indonesian": 0.7333333333333333, "Spanish,English": 0.7733333333333333, "Malay,Indonesian": 0.76, "Malay,English": 0.76, "Indonesian,English": 0.7333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.49333333333333335, "Filipino,Vietnamese,Spanish": 0.5733333333333334, "Filipino,Vietnamese,Malay": 0.6, "Filipino,Vietnamese,Indonesian": 0.5866666666666667, "Filipino,Vietnamese,English": 0.6133333333333333, "Filipino,Chinese,Spanish": 0.5133333333333333, "Filipino,Chinese,Malay": 0.4866666666666667, "Filipino,Chinese,Indonesian": 0.49333333333333335, "Filipino,Chinese,English": 0.5333333333333333, "Filipino,Spanish,Malay": 0.6133333333333333, "Filipino,Spanish,Indonesian": 0.6, "Filipino,Spanish,English": 0.6666666666666666, "Filipino,Malay,Indonesian": 0.62, "Filipino,Malay,English": 0.6466666666666666, "Filipino,Indonesian,English": 0.62, "Vietnamese,Chinese,Spanish": 0.5066666666666667, "Vietnamese,Chinese,Malay": 0.4866666666666667, "Vietnamese,Chinese,Indonesian": 0.52, "Vietnamese,Chinese,English": 0.5133333333333333, "Vietnamese,Spanish,Malay": 0.5666666666666667, "Vietnamese,Spanish,Indonesian": 0.5866666666666667, "Vietnamese,Spanish,English": 0.6066666666666667, "Vietnamese,Malay,Indonesian": 0.6066666666666667, "Vietnamese,Malay,English": 0.6066666666666667, "Vietnamese,Indonesian,English": 0.5933333333333334, "Chinese,Spanish,Malay": 0.5, "Chinese,Spanish,Indonesian": 0.5466666666666666, "Chinese,Spanish,English": 0.5466666666666666, "Chinese,Malay,Indonesian": 0.52, "Chinese,Malay,English": 0.5066666666666667, "Chinese,Indonesian,English": 0.5333333333333333, "Spanish,Malay,Indonesian": 0.6333333333333333, "Spanish,Malay,English": 0.6533333333333333, "Spanish,Indonesian,English": 0.64, "Malay,Indonesian,English": 0.6466666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.44, "Filipino,Vietnamese,Chinese,Malay": 0.44, "Filipino,Vietnamese,Chinese,Indonesian": 0.43333333333333335, "Filipino,Vietnamese,Chinese,English": 0.46, "Filipino,Vietnamese,Spanish,Malay": 0.5133333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.52, "Filipino,Vietnamese,Spanish,English": 0.5466666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.54, "Filipino,Vietnamese,Malay,English": 0.5533333333333333, "Filipino,Vietnamese,Indonesian,English": 0.5333333333333333, "Filipino,Chinese,Spanish,Malay": 0.43333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.4533333333333333, "Filipino,Chinese,Spanish,English": 0.48, "Filipino,Chinese,Malay,Indonesian": 0.4533333333333333, "Filipino,Chinese,Malay,English": 0.4533333333333333, "Filipino,Chinese,Indonesian,English": 0.46, "Filipino,Spanish,Malay,Indonesian": 0.5466666666666666, "Filipino,Spanish,Malay,English": 0.58, "Filipino,Spanish,Indonesian,English": 0.5666666666666667, "Filipino,Malay,Indonesian,English": 0.5666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.44, "Vietnamese,Chinese,Spanish,Indonesian": 0.4533333333333333, "Vietnamese,Chinese,Spanish,English": 0.46, "Vietnamese,Chinese,Malay,Indonesian": 0.4533333333333333, "Vietnamese,Chinese,Malay,English": 0.44666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.44666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.54, "Vietnamese,Spanish,Malay,English": 0.54, "Vietnamese,Spanish,Indonesian,English": 0.5466666666666666, "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay,Indonesian": 0.48, "Chinese,Spanish,Malay,English": 0.46, "Chinese,Spanish,Indonesian,English": 0.4866666666666667, "Chinese,Malay,Indonesian,English": 0.47333333333333333, "Spanish,Malay,Indonesian,English": 0.5866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.4, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4, "Filipino,Vietnamese,Chinese,Spanish,English": 0.42, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.42, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4066666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.5, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5, "Filipino,Vietnamese,Malay,Indonesian,English": 0.5066666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.42, "Filipino,Chinese,Spanish,Malay,English": 0.41333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.43333333333333335, "Filipino,Chinese,Malay,Indonesian,English": 0.4266666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.52, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.42, "Vietnamese,Chinese,Malay,Indonesian,English": 0.42, "Vietnamese,Spanish,Malay,Indonesian,English": 0.5133333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.3933333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.47333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.4, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.37333333333333335 } }, "AC3_2": 0.5481988949799219, "AC3_3": 0.5041667331615745, "AC3_4": 0.47240624186751473, "AC3_5": 0.4472635100382127, "AC3_6": 0.426293350252414, "AC3_7": 0.40868360272181087 }, "prompt_2": { "overall_acc": 0.44476190476190475, "language_acc": { "Filipino": 0.4266666666666667, "Vietnamese": 0.44, "Chinese": 0.46, "Spanish": 0.4, "Malay": 0.4866666666666667, "Indonesian": 0.43333333333333335, "English": 0.4666666666666667 }, "consistency_score_2": 0.7069841269841268, "consistency_score_3": 0.584190476190476, "consistency_score_4": 0.5085714285714285, "consistency_score_5": 0.45396825396825397, "consistency_score_6": 0.41238095238095235, "consistency_score_7": 0.38, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.7066666666666667, "Filipino,Chinese": 0.62, "Filipino,Spanish": 0.7466666666666667, "Filipino,Malay": 0.74, "Filipino,Indonesian": 0.7333333333333333, "Filipino,English": 0.7666666666666667, "Vietnamese,Chinese": 0.68, "Vietnamese,Spanish": 0.66, "Vietnamese,Malay": 0.7133333333333334, "Vietnamese,Indonesian": 0.7266666666666667, "Vietnamese,English": 0.7, "Chinese,Spanish": 0.64, "Chinese,Malay": 0.6133333333333333, "Chinese,Indonesian": 0.6666666666666666, "Chinese,English": 0.6466666666666666, "Spanish,Malay": 0.7333333333333333, "Spanish,Indonesian": 0.7133333333333334, "Spanish,English": 0.74, "Malay,Indonesian": 0.78, "Malay,English": 0.7733333333333333, "Indonesian,English": 0.7466666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5266666666666666, "Filipino,Vietnamese,Spanish": 0.58, "Filipino,Vietnamese,Malay": 0.6066666666666667, "Filipino,Vietnamese,Indonesian": 0.62, "Filipino,Vietnamese,English": 0.6066666666666667, "Filipino,Chinese,Spanish": 0.5333333333333333, "Filipino,Chinese,Malay": 0.5, "Filipino,Chinese,Indonesian": 0.5333333333333333, "Filipino,Chinese,English": 0.5466666666666666, "Filipino,Spanish,Malay": 0.6266666666666667, "Filipino,Spanish,Indonesian": 0.6266666666666667, "Filipino,Spanish,English": 0.64, "Filipino,Malay,Indonesian": 0.6466666666666666, "Filipino,Malay,English": 0.6533333333333333, "Filipino,Indonesian,English": 0.6466666666666666, "Vietnamese,Chinese,Spanish": 0.5266666666666666, "Vietnamese,Chinese,Malay": 0.5333333333333333, "Vietnamese,Chinese,Indonesian": 0.5666666666666667, "Vietnamese,Chinese,English": 0.5466666666666666, "Vietnamese,Spanish,Malay": 0.5733333333333334, "Vietnamese,Spanish,Indonesian": 0.5866666666666667, "Vietnamese,Spanish,English": 0.5866666666666667, "Vietnamese,Malay,Indonesian": 0.6333333333333333, "Vietnamese,Malay,English": 0.6066666666666667, "Vietnamese,Indonesian,English": 0.6133333333333333, "Chinese,Spanish,Malay": 0.5066666666666667, "Chinese,Spanish,Indonesian": 0.54, "Chinese,Spanish,English": 0.54, "Chinese,Malay,Indonesian": 0.5466666666666666, "Chinese,Malay,English": 0.54, "Chinese,Indonesian,English": 0.56, "Spanish,Malay,Indonesian": 0.6333333333333333, "Spanish,Malay,English": 0.6333333333333333, "Spanish,Indonesian,English": 0.62, "Malay,Indonesian,English": 0.66 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.4666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.46, "Filipino,Vietnamese,Chinese,Indonesian": 0.48, "Filipino,Vietnamese,Chinese,English": 0.48, "Filipino,Vietnamese,Spanish,Malay": 0.5266666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.5333333333333333, "Filipino,Vietnamese,Spanish,English": 0.5333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.5733333333333334, "Filipino,Vietnamese,Malay,English": 0.5533333333333333, "Filipino,Vietnamese,Indonesian,English": 0.56, "Filipino,Chinese,Spanish,Malay": 0.44, "Filipino,Chinese,Spanish,Indonesian": 0.47333333333333333, "Filipino,Chinese,Spanish,English": 0.4866666666666667, "Filipino,Chinese,Malay,Indonesian": 0.4666666666666667, "Filipino,Chinese,Malay,English": 0.47333333333333333, "Filipino,Chinese,Indonesian,English": 0.49333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.56, "Filipino,Spanish,Malay,English": 0.5733333333333334, "Filipino,Spanish,Indonesian,English": 0.5666666666666667, "Filipino,Malay,Indonesian,English": 0.5866666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, "Vietnamese,Chinese,Spanish,English": 0.47333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.49333333333333335, "Vietnamese,Chinese,Malay,English": 0.48, "Vietnamese,Chinese,Indonesian,English": 0.5, "Vietnamese,Spanish,Malay,Indonesian": 0.54, "Vietnamese,Spanish,Malay,English": 0.52, "Vietnamese,Spanish,Indonesian,English": 0.5333333333333333, "Vietnamese,Malay,Indonesian,English": 0.5666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.4666666666666667, "Chinese,Spanish,Malay,English": 0.46, "Chinese,Spanish,Indonesian,English": 0.49333333333333335, "Chinese,Malay,Indonesian,English": 0.5, "Spanish,Malay,Indonesian,English": 0.56 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.44, "Filipino,Vietnamese,Chinese,Malay,English": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.44666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.5, "Filipino,Vietnamese,Spanish,Malay,English": 0.49333333333333335, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5, "Filipino,Vietnamese,Malay,Indonesian,English": 0.5266666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.42, "Filipino,Chinese,Spanish,Indonesian,English": 0.44666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.44666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.52, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.42, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.44, "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian,English": 0.49333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3933333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.3933333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.42, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.47333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.4, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38 } }, "AC3_2": 0.5460224706828906, "AC3_3": 0.5050295304599006, "AC3_4": 0.47453118305283565, "AC3_5": 0.4493179256367052, "AC3_6": 0.4279597883098597, "AC3_7": 0.4098383371327563 }, "prompt_3": { "overall_acc": 0.4371428571428571, "language_acc": { "Filipino": 0.4066666666666667, "Vietnamese": 0.4533333333333333, "Chinese": 0.47333333333333333, "Spanish": 0.37333333333333335, "Malay": 0.47333333333333333, "Indonesian": 0.4266666666666667, "English": 0.4533333333333333 }, "consistency_score_2": 0.6971428571428572, "consistency_score_3": 0.5697142857142856, "consistency_score_4": 0.4956190476190476, "consistency_score_5": 0.4453968253968254, "consistency_score_6": 0.40857142857142853, "consistency_score_7": 0.38, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6666666666666666, "Filipino,Chinese": 0.6266666666666667, "Filipino,Spanish": 0.72, "Filipino,Malay": 0.72, "Filipino,Indonesian": 0.6933333333333334, "Filipino,English": 0.7666666666666667, "Vietnamese,Chinese": 0.68, "Vietnamese,Spanish": 0.64, "Vietnamese,Malay": 0.7, "Vietnamese,Indonesian": 0.7333333333333333, "Vietnamese,English": 0.7133333333333334, "Chinese,Spanish": 0.6333333333333333, "Chinese,Malay": 0.58, "Chinese,Indonesian": 0.6866666666666666, "Chinese,English": 0.6933333333333334, "Spanish,Malay": 0.7, "Spanish,Indonesian": 0.7266666666666667, "Spanish,English": 0.74, "Malay,Indonesian": 0.7666666666666667, "Malay,English": 0.76, "Indonesian,English": 0.6933333333333334 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5066666666666667, "Filipino,Vietnamese,Spanish": 0.54, "Filipino,Vietnamese,Malay": 0.5733333333333334, "Filipino,Vietnamese,Indonesian": 0.5866666666666667, "Filipino,Vietnamese,English": 0.5933333333333334, "Filipino,Chinese,Spanish": 0.5133333333333333, "Filipino,Chinese,Malay": 0.4866666666666667, "Filipino,Chinese,Indonesian": 0.5266666666666666, "Filipino,Chinese,English": 0.5666666666666667, "Filipino,Spanish,Malay": 0.5933333333333334, "Filipino,Spanish,Indonesian": 0.5933333333333334, "Filipino,Spanish,English": 0.6266666666666667, "Filipino,Malay,Indonesian": 0.6133333333333333, "Filipino,Malay,English": 0.6333333333333333, "Filipino,Indonesian,English": 0.6066666666666667, "Vietnamese,Chinese,Spanish": 0.5066666666666667, "Vietnamese,Chinese,Malay": 0.5066666666666667, "Vietnamese,Chinese,Indonesian": 0.5733333333333334, "Vietnamese,Chinese,English": 0.5666666666666667, "Vietnamese,Spanish,Malay": 0.54, "Vietnamese,Spanish,Indonesian": 0.58, "Vietnamese,Spanish,English": 0.58, "Vietnamese,Malay,Indonesian": 0.6266666666666667, "Vietnamese,Malay,English": 0.6, "Vietnamese,Indonesian,English": 0.6, "Chinese,Spanish,Malay": 0.48, "Chinese,Spanish,Indonesian": 0.56, "Chinese,Spanish,English": 0.5533333333333333, "Chinese,Malay,Indonesian": 0.54, "Chinese,Malay,English": 0.54, "Chinese,Indonesian,English": 0.5666666666666667, "Spanish,Malay,Indonesian": 0.6133333333333333, "Spanish,Malay,English": 0.6133333333333333, "Spanish,Indonesian,English": 0.6066666666666667, "Malay,Indonesian,English": 0.6266666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Malay": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Indonesian": 0.4533333333333333, "Filipino,Vietnamese,Chinese,English": 0.47333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.49333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.5066666666666667, "Filipino,Vietnamese,Spanish,English": 0.5066666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.5466666666666666, "Filipino,Vietnamese,Malay,English": 0.54, "Filipino,Vietnamese,Indonesian,English": 0.5333333333333333, "Filipino,Chinese,Spanish,Malay": 0.4266666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.4666666666666667, "Filipino,Chinese,Spanish,English": 0.4866666666666667, "Filipino,Chinese,Malay,Indonesian": 0.4666666666666667, "Filipino,Chinese,Malay,English": 0.48, "Filipino,Chinese,Indonesian,English": 0.49333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.54, "Filipino,Spanish,Malay,English": 0.5533333333333333, "Filipino,Spanish,Indonesian,English": 0.5466666666666666, "Filipino,Malay,Indonesian,English": 0.5666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.42, "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.48, "Vietnamese,Chinese,Malay,English": 0.47333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, "Vietnamese,Spanish,Malay,Indonesian": 0.5133333333333333, "Vietnamese,Spanish,Malay,English": 0.5133333333333333, "Vietnamese,Spanish,Indonesian,English": 0.5266666666666666, "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay,Indonesian": 0.4666666666666667, "Chinese,Spanish,Malay,English": 0.46, "Chinese,Spanish,Indonesian,English": 0.5066666666666667, "Chinese,Malay,Indonesian,English": 0.5066666666666667, "Spanish,Malay,Indonesian,English": 0.5466666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.42, "Filipino,Vietnamese,Chinese,Malay,English": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4266666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.48, "Filipino,Vietnamese,Spanish,Malay,English": 0.48, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.48, "Filipino,Vietnamese,Malay,Indonesian,English": 0.5133333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.42, "Filipino,Chinese,Spanish,Malay,English": 0.4266666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.4533333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.46, "Filipino,Spanish,Malay,Indonesian,English": 0.5133333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.44, "Vietnamese,Chinese,Malay,Indonesian,English": 0.4533333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4866666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.3933333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.41333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.4666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.42, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38 } }, "AC3_2": 0.5373443684304968, "AC3_3": 0.49470082693319206, "AC3_4": 0.46454797392843394, "AC3_5": 0.44123124352657167, "AC3_6": 0.42237451732457443, "AC3_7": 0.40657342652367107 }, "prompt_4": { "overall_acc": 0.439047619047619, "language_acc": { "Filipino": 0.4266666666666667, "Vietnamese": 0.46, "Chinese": 0.43333333333333335, "Spanish": 0.4, "Malay": 0.48, "Indonesian": 0.41333333333333333, "English": 0.46 }, "consistency_score_2": 0.6946031746031746, "consistency_score_3": 0.5664761904761904, "consistency_score_4": 0.49085714285714277, "consistency_score_5": 0.43936507936507946, "consistency_score_6": 0.40190476190476193, "consistency_score_7": 0.37333333333333335, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6733333333333333, "Filipino,Chinese": 0.6, "Filipino,Spanish": 0.7333333333333333, "Filipino,Malay": 0.6933333333333334, "Filipino,Indonesian": 0.68, "Filipino,English": 0.7666666666666667, "Vietnamese,Chinese": 0.68, "Vietnamese,Spanish": 0.66, "Vietnamese,Malay": 0.7066666666666667, "Vietnamese,Indonesian": 0.72, "Vietnamese,English": 0.72, "Chinese,Spanish": 0.6266666666666667, "Chinese,Malay": 0.5733333333333334, "Chinese,Indonesian": 0.6666666666666666, "Chinese,English": 0.66, "Spanish,Malay": 0.6933333333333334, "Spanish,Indonesian": 0.74, "Spanish,English": 0.76, "Malay,Indonesian": 0.7266666666666667, "Malay,English": 0.76, "Indonesian,English": 0.7466666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5066666666666667, "Filipino,Vietnamese,Spanish": 0.56, "Filipino,Vietnamese,Malay": 0.5733333333333334, "Filipino,Vietnamese,Indonesian": 0.5733333333333334, "Filipino,Vietnamese,English": 0.5933333333333334, "Filipino,Chinese,Spanish": 0.5133333333333333, "Filipino,Chinese,Malay": 0.4533333333333333, "Filipino,Chinese,Indonesian": 0.5066666666666667, "Filipino,Chinese,English": 0.5333333333333333, "Filipino,Spanish,Malay": 0.5866666666666667, "Filipino,Spanish,Indonesian": 0.6, "Filipino,Spanish,English": 0.6466666666666666, "Filipino,Malay,Indonesian": 0.58, "Filipino,Malay,English": 0.6266666666666667, "Filipino,Indonesian,English": 0.62, "Vietnamese,Chinese,Spanish": 0.52, "Vietnamese,Chinese,Malay": 0.5133333333333333, "Vietnamese,Chinese,Indonesian": 0.5533333333333333, "Vietnamese,Chinese,English": 0.5533333333333333, "Vietnamese,Spanish,Malay": 0.5466666666666666, "Vietnamese,Spanish,Indonesian": 0.5933333333333334, "Vietnamese,Spanish,English": 0.5933333333333334, "Vietnamese,Malay,Indonesian": 0.6066666666666667, "Vietnamese,Malay,English": 0.6066666666666667, "Vietnamese,Indonesian,English": 0.62, "Chinese,Spanish,Malay": 0.4666666666666667, "Chinese,Spanish,Indonesian": 0.5466666666666666, "Chinese,Spanish,English": 0.54, "Chinese,Malay,Indonesian": 0.5066666666666667, "Chinese,Malay,English": 0.5133333333333333, "Chinese,Indonesian,English": 0.56, "Spanish,Malay,Indonesian": 0.6066666666666667, "Spanish,Malay,English": 0.6266666666666667, "Spanish,Indonesian,English": 0.6533333333333333, "Malay,Indonesian,English": 0.6266666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.4533333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Indonesian": 0.4533333333333333, "Filipino,Vietnamese,Chinese,English": 0.47333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.49333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,Spanish,English": 0.5266666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.52, "Filipino,Vietnamese,Malay,English": 0.5333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.5333333333333333, "Filipino,Chinese,Spanish,Malay": 0.4066666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.4533333333333333, "Filipino,Chinese,Spanish,English": 0.47333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.42, "Filipino,Chinese,Malay,English": 0.44, "Filipino,Chinese,Indonesian,English": 0.48, "Filipino,Spanish,Malay,Indonesian": 0.52, "Filipino,Spanish,Malay,English": 0.5533333333333333, "Filipino,Spanish,Indonesian,English": 0.5666666666666667, "Filipino,Malay,Indonesian,English": 0.54, "Vietnamese,Chinese,Spanish,Malay": 0.43333333333333335, "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, "Vietnamese,Chinese,Spanish,English": 0.48, "Vietnamese,Chinese,Malay,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,Malay,English": 0.47333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, "Vietnamese,Spanish,Malay,Indonesian": 0.52, "Vietnamese,Spanish,Malay,English": 0.5133333333333333, "Vietnamese,Spanish,Indonesian,English": 0.5533333333333333, "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Chinese,Spanish,Malay,English": 0.44666666666666666, "Chinese,Spanish,Indonesian,English": 0.5066666666666667, "Chinese,Malay,Indonesian,English": 0.47333333333333333, "Spanish,Malay,Indonesian,English": 0.56 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.3933333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.42, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.43333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.47333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.49333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.49333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.38666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.4, "Filipino,Chinese,Spanish,Indonesian,English": 0.44, "Filipino,Chinese,Malay,Indonesian,English": 0.41333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.5, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.42, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4533333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.44666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.49333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.4, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.4533333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4066666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.37333333333333335 } }, "AC3_2": 0.5380208291187075, "AC3_3": 0.4946874858563225, "AC3_4": 0.46350910043590454, "AC3_5": 0.4392062917909142, "AC3_6": 0.41965593480422037, "AC3_7": 0.40353262988389493 }, "prompt_5": { "overall_acc": 0.43619047619047624, "language_acc": { "Filipino": 0.44, "Vietnamese": 0.44, "Chinese": 0.44666666666666666, "Spanish": 0.38666666666666666, "Malay": 0.4666666666666667, "Indonesian": 0.4066666666666667, "English": 0.4666666666666667 }, "consistency_score_2": 0.7000000000000002, "consistency_score_3": 0.5779047619047618, "consistency_score_4": 0.5095238095238095, "consistency_score_5": 0.4647619047619048, "consistency_score_6": 0.4323809523809525, "consistency_score_7": 0.4066666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6733333333333333, "Filipino,Chinese": 0.6066666666666667, "Filipino,Spanish": 0.7066666666666667, "Filipino,Malay": 0.7333333333333333, "Filipino,Indonesian": 0.6933333333333334, "Filipino,English": 0.7533333333333333, "Vietnamese,Chinese": 0.6533333333333333, "Vietnamese,Spanish": 0.6533333333333333, "Vietnamese,Malay": 0.7066666666666667, "Vietnamese,Indonesian": 0.7266666666666667, "Vietnamese,English": 0.72, "Chinese,Spanish": 0.6533333333333333, "Chinese,Malay": 0.5733333333333334, "Chinese,Indonesian": 0.6466666666666666, "Chinese,English": 0.66, "Spanish,Malay": 0.7266666666666667, "Spanish,Indonesian": 0.7533333333333333, "Spanish,English": 0.7666666666666667, "Malay,Indonesian": 0.7666666666666667, "Malay,English": 0.78, "Indonesian,English": 0.7466666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5066666666666667, "Filipino,Vietnamese,Spanish": 0.5533333333333333, "Filipino,Vietnamese,Malay": 0.5866666666666667, "Filipino,Vietnamese,Indonesian": 0.5933333333333334, "Filipino,Vietnamese,English": 0.6, "Filipino,Chinese,Spanish": 0.52, "Filipino,Chinese,Malay": 0.48, "Filipino,Chinese,Indonesian": 0.52, "Filipino,Chinese,English": 0.54, "Filipino,Spanish,Malay": 0.6066666666666667, "Filipino,Spanish,Indonesian": 0.6, "Filipino,Spanish,English": 0.6266666666666667, "Filipino,Malay,Indonesian": 0.62, "Filipino,Malay,English": 0.64, "Filipino,Indonesian,English": 0.6133333333333333, "Vietnamese,Chinese,Spanish": 0.52, "Vietnamese,Chinese,Malay": 0.5066666666666667, "Vietnamese,Chinese,Indonesian": 0.5466666666666666, "Vietnamese,Chinese,English": 0.54, "Vietnamese,Spanish,Malay": 0.5666666666666667, "Vietnamese,Spanish,Indonesian": 0.6, "Vietnamese,Spanish,English": 0.6, "Vietnamese,Malay,Indonesian": 0.6333333333333333, "Vietnamese,Malay,English": 0.62, "Vietnamese,Indonesian,English": 0.6333333333333333, "Chinese,Spanish,Malay": 0.5066666666666667, "Chinese,Spanish,Indonesian": 0.5533333333333333, "Chinese,Spanish,English": 0.56, "Chinese,Malay,Indonesian": 0.5333333333333333, "Chinese,Malay,English": 0.5333333333333333, "Chinese,Indonesian,English": 0.56, "Spanish,Malay,Indonesian": 0.6333333333333333, "Spanish,Malay,English": 0.6533333333333333, "Spanish,Indonesian,English": 0.66, "Malay,Indonesian,English": 0.66 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.46, "Filipino,Vietnamese,Chinese,Malay": 0.4533333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.47333333333333333, "Filipino,Vietnamese,Chinese,English": 0.48, "Filipino,Vietnamese,Spanish,Malay": 0.5133333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,Spanish,English": 0.5333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.56, "Filipino,Vietnamese,Malay,English": 0.5466666666666666, "Filipino,Vietnamese,Indonesian,English": 0.5466666666666666, "Filipino,Chinese,Spanish,Malay": 0.44666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.48, "Filipino,Chinese,Spanish,English": 0.4866666666666667, "Filipino,Chinese,Malay,Indonesian": 0.47333333333333333, "Filipino,Chinese,Malay,English": 0.4666666666666667, "Filipino,Chinese,Indonesian,English": 0.49333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.5533333333333333, "Filipino,Spanish,Malay,English": 0.5666666666666667, "Filipino,Spanish,Indonesian,English": 0.5666666666666667, "Filipino,Malay,Indonesian,English": 0.5666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.48, "Vietnamese,Chinese,Spanish,English": 0.48, "Vietnamese,Chinese,Malay,Indonesian": 0.48, "Vietnamese,Chinese,Malay,English": 0.47333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, "Vietnamese,Spanish,Malay,Indonesian": 0.54, "Vietnamese,Spanish,Malay,English": 0.54, "Vietnamese,Spanish,Indonesian,English": 0.56, "Vietnamese,Malay,Indonesian,English": 0.5733333333333334, "Chinese,Spanish,Malay,Indonesian": 0.48, "Chinese,Spanish,Malay,English": 0.49333333333333335, "Chinese,Spanish,Indonesian,English": 0.5133333333333333, "Chinese,Malay,Indonesian,English": 0.5066666666666667, "Spanish,Malay,Indonesian,English": 0.5866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.42, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Spanish,English": 0.44666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.44666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.44, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.49333333333333335, "Filipino,Vietnamese,Spanish,Malay,English": 0.5, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5, "Filipino,Vietnamese,Malay,Indonesian,English": 0.52, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.44, "Filipino,Chinese,Spanish,Malay,English": 0.44, "Filipino,Chinese,Spanish,Indonesian,English": 0.4666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.46, "Filipino,Spanish,Malay,Indonesian,English": 0.5266666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.44, "Vietnamese,Chinese,Spanish,Malay,English": 0.44, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4533333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian,English": 0.5133333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.43333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.48, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4266666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4066666666666667 } }, "AC3_2": 0.5374685665914216, "AC3_3": 0.4971457192454561, "AC3_4": 0.47001390682221356, "AC3_5": 0.45002315508948454, "AC3_6": 0.4342773600168347, "AC3_7": 0.4209114877090067 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.4862012987012987, "language_acc": { "Indonesian": 0.4715909090909091, "English": 0.5056818181818182, "Filipino": 0.5113636363636364, "Spanish": 0.4772727272727273, "Chinese": 0.48863636363636365, "Malay": 0.4772727272727273, "Vietnamese": 0.4715909090909091 }, "consistency_score_2": 0.7167207792207793, "consistency_score_3": 0.5917207792207791, "consistency_score_4": 0.5137987012987014, "consistency_score_5": 0.4583333333333333, "consistency_score_6": 0.4155844155844156, "consistency_score_7": 0.3806818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7329545454545454, "Indonesian,Filipino": 0.7272727272727273, "Indonesian,Spanish": 0.7159090909090909, "Indonesian,Chinese": 0.6761363636363636, "Indonesian,Malay": 0.7159090909090909, "Indonesian,Vietnamese": 0.7613636363636364, "English,Filipino": 0.75, "English,Spanish": 0.7897727272727273, "English,Chinese": 0.6761363636363636, "English,Malay": 0.7102272727272727, "English,Vietnamese": 0.7954545454545454, "Filipino,Spanish": 0.7443181818181818, "Filipino,Chinese": 0.6590909090909091, "Filipino,Malay": 0.6704545454545454, "Filipino,Vietnamese": 0.7159090909090909, "Spanish,Chinese": 0.6306818181818182, "Spanish,Malay": 0.7386363636363636, "Spanish,Vietnamese": 0.7727272727272727, "Chinese,Malay": 0.6363636363636364, "Chinese,Vietnamese": 0.6931818181818182, "Malay,Vietnamese": 0.7386363636363636 }, "3_combine": { "Indonesian,English,Filipino": 0.6136363636363636, "Indonesian,English,Spanish": 0.6306818181818182, "Indonesian,English,Chinese": 0.5568181818181818, "Indonesian,English,Malay": 0.5965909090909091, "Indonesian,English,Vietnamese": 0.6590909090909091, "Indonesian,Filipino,Spanish": 0.6079545454545454, "Indonesian,Filipino,Chinese": 0.5454545454545454, "Indonesian,Filipino,Malay": 0.5738636363636364, "Indonesian,Filipino,Vietnamese": 0.6193181818181818, "Indonesian,Spanish,Chinese": 0.5227272727272727, "Indonesian,Spanish,Malay": 0.6022727272727273, "Indonesian,Spanish,Vietnamese": 0.6363636363636364, "Indonesian,Chinese,Malay": 0.5340909090909091, "Indonesian,Chinese,Vietnamese": 0.5852272727272727, "Indonesian,Malay,Vietnamese": 0.6363636363636364, "English,Filipino,Spanish": 0.6477272727272727, "English,Filipino,Chinese": 0.5568181818181818, "English,Filipino,Malay": 0.5795454545454546, "English,Filipino,Vietnamese": 0.6420454545454546, "English,Spanish,Chinese": 0.5681818181818182, "English,Spanish,Malay": 0.6306818181818182, "English,Spanish,Vietnamese": 0.6931818181818182, "English,Chinese,Malay": 0.5284090909090909, "English,Chinese,Vietnamese": 0.6022727272727273, "English,Malay,Vietnamese": 0.6477272727272727, "Filipino,Spanish,Chinese": 0.5340909090909091, "Filipino,Spanish,Malay": 0.5965909090909091, "Filipino,Spanish,Vietnamese": 0.6306818181818182, "Filipino,Chinese,Malay": 0.5056818181818182, "Filipino,Chinese,Vietnamese": 0.5511363636363636, "Filipino,Malay,Vietnamese": 0.5909090909090909, "Spanish,Chinese,Malay": 0.5170454545454546, "Spanish,Chinese,Vietnamese": 0.5625, "Spanish,Malay,Vietnamese": 0.6420454545454546, "Chinese,Malay,Vietnamese": 0.5625 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.5568181818181818, "Indonesian,English,Filipino,Chinese": 0.48863636363636365, "Indonesian,English,Filipino,Malay": 0.5056818181818182, "Indonesian,English,Filipino,Vietnamese": 0.5681818181818182, "Indonesian,English,Spanish,Chinese": 0.48863636363636365, "Indonesian,English,Spanish,Malay": 0.5454545454545454, "Indonesian,English,Spanish,Vietnamese": 0.5909090909090909, "Indonesian,English,Chinese,Malay": 0.4659090909090909, "Indonesian,English,Chinese,Vietnamese": 0.5340909090909091, "Indonesian,English,Malay,Vietnamese": 0.5568181818181818, "Indonesian,Filipino,Spanish,Chinese": 0.45454545454545453, "Indonesian,Filipino,Spanish,Malay": 0.5170454545454546, "Indonesian,Filipino,Spanish,Vietnamese": 0.5625, "Indonesian,Filipino,Chinese,Malay": 0.4431818181818182, "Indonesian,Filipino,Chinese,Vietnamese": 0.48295454545454547, "Indonesian,Filipino,Malay,Vietnamese": 0.5284090909090909, "Indonesian,Spanish,Chinese,Malay": 0.45454545454545453, "Indonesian,Spanish,Chinese,Vietnamese": 0.4943181818181818, "Indonesian,Spanish,Malay,Vietnamese": 0.5568181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.5, "English,Filipino,Spanish,Chinese": 0.4943181818181818, "English,Filipino,Spanish,Malay": 0.5340909090909091, "English,Filipino,Spanish,Vietnamese": 0.5852272727272727, "English,Filipino,Chinese,Malay": 0.45454545454545453, "English,Filipino,Chinese,Vietnamese": 0.5170454545454546, "English,Filipino,Malay,Vietnamese": 0.5397727272727273, "English,Spanish,Chinese,Malay": 0.4715909090909091, "English,Spanish,Chinese,Vietnamese": 0.5340909090909091, "English,Spanish,Malay,Vietnamese": 0.5909090909090909, "English,Chinese,Malay,Vietnamese": 0.5170454545454546, "Filipino,Spanish,Chinese,Malay": 0.45454545454545453, "Filipino,Spanish,Chinese,Vietnamese": 0.48863636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.5397727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.4715909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.4943181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4375, "Indonesian,English,Filipino,Spanish,Malay": 0.4772727272727273, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5284090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.4090909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.48295454545454547, "Indonesian,English,Spanish,Chinese,Malay": 0.42613636363636365, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4772727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5170454545454546, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4602272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3977272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4375, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.48863636363636365, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.42045454545454547, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4431818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4715909090909091, "English,Filipino,Spanish,Malay,Vietnamese": 0.5056818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.44886363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.4715909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4375 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3806818181818182, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4602272727272727, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.4034090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.39204545454545453, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3806818181818182 } }, "AC3_2": 0.5793734773516294, "AC3_3": 0.5337963053017544, "AC3_4": 0.49961919163497664, "AC3_5": 0.4718562016114586, "AC3_6": 0.44812792962639114, "AC3_7": 0.4270194926303459 }, "prompt_2": { "overall_acc": 0.49350649350649345, "language_acc": { "Indonesian": 0.48295454545454547, "English": 0.5227272727272727, "Filipino": 0.4943181818181818, "Spanish": 0.48863636363636365, "Chinese": 0.4943181818181818, "Malay": 0.4943181818181818, "Vietnamese": 0.4772727272727273 }, "consistency_score_2": 0.7088744588744588, "consistency_score_3": 0.5788961038961039, "consistency_score_4": 0.5003246753246753, "consistency_score_5": 0.4458874458874459, "consistency_score_6": 0.40422077922077915, "consistency_score_7": 0.3693181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7159090909090909, "Indonesian,Filipino": 0.7159090909090909, "Indonesian,Spanish": 0.7159090909090909, "Indonesian,Chinese": 0.6647727272727273, "Indonesian,Malay": 0.7556818181818182, "Indonesian,Vietnamese": 0.7556818181818182, "English,Filipino": 0.75, "English,Spanish": 0.7727272727272727, "English,Chinese": 0.6818181818181818, "English,Malay": 0.6818181818181818, "English,Vietnamese": 0.7556818181818182, "Filipino,Spanish": 0.7159090909090909, "Filipino,Chinese": 0.6704545454545454, "Filipino,Malay": 0.7045454545454546, "Filipino,Vietnamese": 0.6988636363636364, "Spanish,Chinese": 0.6193181818181818, "Spanish,Malay": 0.7329545454545454, "Spanish,Vietnamese": 0.7556818181818182, "Chinese,Malay": 0.6306818181818182, "Chinese,Vietnamese": 0.6590909090909091, "Malay,Vietnamese": 0.7329545454545454 }, "3_combine": { "Indonesian,English,Filipino": 0.6022727272727273, "Indonesian,English,Spanish": 0.6079545454545454, "Indonesian,English,Chinese": 0.5397727272727273, "Indonesian,English,Malay": 0.5909090909090909, "Indonesian,English,Vietnamese": 0.625, "Indonesian,Filipino,Spanish": 0.5852272727272727, "Indonesian,Filipino,Chinese": 0.5397727272727273, "Indonesian,Filipino,Malay": 0.6079545454545454, "Indonesian,Filipino,Vietnamese": 0.6022727272727273, "Indonesian,Spanish,Chinese": 0.5113636363636364, "Indonesian,Spanish,Malay": 0.6136363636363636, "Indonesian,Spanish,Vietnamese": 0.625, "Indonesian,Chinese,Malay": 0.5454545454545454, "Indonesian,Chinese,Vietnamese": 0.5625, "Indonesian,Malay,Vietnamese": 0.6420454545454546, "English,Filipino,Spanish": 0.625, "English,Filipino,Chinese": 0.5681818181818182, "English,Filipino,Malay": 0.5795454545454546, "English,Filipino,Vietnamese": 0.6193181818181818, "English,Spanish,Chinese": 0.5511363636363636, "English,Spanish,Malay": 0.6022727272727273, "English,Spanish,Vietnamese": 0.6590909090909091, "English,Chinese,Malay": 0.5113636363636364, "English,Chinese,Vietnamese": 0.5681818181818182, "English,Malay,Vietnamese": 0.6079545454545454, "Filipino,Spanish,Chinese": 0.5284090909090909, "Filipino,Spanish,Malay": 0.5909090909090909, "Filipino,Spanish,Vietnamese": 0.6079545454545454, "Filipino,Chinese,Malay": 0.5284090909090909, "Filipino,Chinese,Vietnamese": 0.5284090909090909, "Filipino,Malay,Vietnamese": 0.5965909090909091, "Spanish,Chinese,Malay": 0.5, "Spanish,Chinese,Vietnamese": 0.5284090909090909, "Spanish,Malay,Vietnamese": 0.625, "Chinese,Malay,Vietnamese": 0.5340909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.5284090909090909, "Indonesian,English,Filipino,Chinese": 0.4772727272727273, "Indonesian,English,Filipino,Malay": 0.5227272727272727, "Indonesian,English,Filipino,Vietnamese": 0.5511363636363636, "Indonesian,English,Spanish,Chinese": 0.4659090909090909, "Indonesian,English,Spanish,Malay": 0.5340909090909091, "Indonesian,English,Spanish,Vietnamese": 0.5681818181818182, "Indonesian,English,Chinese,Malay": 0.4659090909090909, "Indonesian,English,Chinese,Vietnamese": 0.5056818181818182, "Indonesian,English,Malay,Vietnamese": 0.5397727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.44886363636363635, "Indonesian,Filipino,Spanish,Malay": 0.5284090909090909, "Indonesian,Filipino,Spanish,Vietnamese": 0.5284090909090909, "Indonesian,Filipino,Chinese,Malay": 0.4715909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.4715909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.5397727272727273, "Indonesian,Spanish,Chinese,Malay": 0.4602272727272727, "Indonesian,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.5454545454545454, "Indonesian,Chinese,Malay,Vietnamese": 0.4943181818181818, "English,Filipino,Spanish,Chinese": 0.4772727272727273, "English,Filipino,Spanish,Malay": 0.5113636363636364, "English,Filipino,Spanish,Vietnamese": 0.5568181818181818, "English,Filipino,Chinese,Malay": 0.4659090909090909, "English,Filipino,Chinese,Vietnamese": 0.4943181818181818, "English,Filipino,Malay,Vietnamese": 0.5284090909090909, "English,Spanish,Chinese,Malay": 0.44886363636363635, "English,Spanish,Chinese,Vietnamese": 0.5, "English,Spanish,Malay,Vietnamese": 0.5568181818181818, "English,Chinese,Malay,Vietnamese": 0.48295454545454547, "Filipino,Spanish,Chinese,Malay": 0.45454545454545453, "Filipino,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.5284090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.4659090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.4602272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4147727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.4715909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5, "Indonesian,English,Filipino,Chinese,Malay": 0.42613636363636365, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.44886363636363635, "Indonesian,English,Filipino,Malay,Vietnamese": 0.48863636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.42045454545454547, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.44886363636363635, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5, "Indonesian,English,Chinese,Malay,Vietnamese": 0.44886363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4147727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.4318181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365, "English,Filipino,Spanish,Chinese,Malay": 0.4090909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Chinese,Malay,Vietnamese": 0.4375, "English,Spanish,Chinese,Malay,Vietnamese": 0.4375, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3806818181818182, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.3977272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.44886363636363635, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.4090909090909091, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818 } }, "AC3_2": 0.5819023460303678, "AC3_3": 0.5328017425613238, "AC3_4": 0.4968921962868246, "AC3_5": 0.46849003526044025, "AC3_6": 0.4444235691480239, "AC3_7": 0.42247498498481667 }, "prompt_3": { "overall_acc": 0.4983766233766234, "language_acc": { "Indonesian": 0.48295454545454547, "English": 0.5284090909090909, "Filipino": 0.4943181818181818, "Spanish": 0.48295454545454547, "Chinese": 0.5056818181818182, "Malay": 0.5284090909090909, "Vietnamese": 0.4659090909090909 }, "consistency_score_2": 0.7110389610389609, "consistency_score_3": 0.5818181818181818, "consistency_score_4": 0.5043831168831169, "consistency_score_5": 0.4510281385281385, "consistency_score_6": 0.4099025974025974, "consistency_score_7": 0.375, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7272727272727273, "Indonesian,Filipino": 0.7272727272727273, "Indonesian,Spanish": 0.7272727272727273, "Indonesian,Chinese": 0.6306818181818182, "Indonesian,Malay": 0.7556818181818182, "Indonesian,Vietnamese": 0.7443181818181818, "English,Filipino": 0.7670454545454546, "English,Spanish": 0.7954545454545454, "English,Chinese": 0.6818181818181818, "English,Malay": 0.7102272727272727, "English,Vietnamese": 0.7670454545454546, "Filipino,Spanish": 0.7215909090909091, "Filipino,Chinese": 0.6704545454545454, "Filipino,Malay": 0.7272727272727273, "Filipino,Vietnamese": 0.6931818181818182, "Spanish,Chinese": 0.6136363636363636, "Spanish,Malay": 0.7443181818181818, "Spanish,Vietnamese": 0.7443181818181818, "Chinese,Malay": 0.6306818181818182, "Chinese,Vietnamese": 0.6363636363636364, "Malay,Vietnamese": 0.7159090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.6193181818181818, "Indonesian,English,Spanish": 0.6306818181818182, "Indonesian,English,Chinese": 0.5284090909090909, "Indonesian,English,Malay": 0.6079545454545454, "Indonesian,English,Vietnamese": 0.6363636363636364, "Indonesian,Filipino,Spanish": 0.5965909090909091, "Indonesian,Filipino,Chinese": 0.5340909090909091, "Indonesian,Filipino,Malay": 0.625, "Indonesian,Filipino,Vietnamese": 0.6022727272727273, "Indonesian,Spanish,Chinese": 0.4943181818181818, "Indonesian,Spanish,Malay": 0.625, "Indonesian,Spanish,Vietnamese": 0.6193181818181818, "Indonesian,Chinese,Malay": 0.5227272727272727, "Indonesian,Chinese,Vietnamese": 0.5227272727272727, "Indonesian,Malay,Vietnamese": 0.6306818181818182, "English,Filipino,Spanish": 0.6477272727272727, "English,Filipino,Chinese": 0.5795454545454546, "English,Filipino,Malay": 0.6193181818181818, "English,Filipino,Vietnamese": 0.6306818181818182, "English,Spanish,Chinese": 0.5568181818181818, "English,Spanish,Malay": 0.6306818181818182, "English,Spanish,Vietnamese": 0.6704545454545454, "English,Chinese,Malay": 0.5227272727272727, "English,Chinese,Vietnamese": 0.5568181818181818, "English,Malay,Vietnamese": 0.6193181818181818, "Filipino,Spanish,Chinese": 0.5227272727272727, "Filipino,Spanish,Malay": 0.6136363636363636, "Filipino,Spanish,Vietnamese": 0.6022727272727273, "Filipino,Chinese,Malay": 0.5340909090909091, "Filipino,Chinese,Vietnamese": 0.5170454545454546, "Filipino,Malay,Vietnamese": 0.6022727272727273, "Spanish,Chinese,Malay": 0.5, "Spanish,Chinese,Vietnamese": 0.5113636363636364, "Spanish,Malay,Vietnamese": 0.6136363636363636, "Chinese,Malay,Vietnamese": 0.5170454545454546 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.5511363636363636, "Indonesian,English,Filipino,Chinese": 0.48295454545454547, "Indonesian,English,Filipino,Malay": 0.5454545454545454, "Indonesian,English,Filipino,Vietnamese": 0.5568181818181818, "Indonesian,English,Spanish,Chinese": 0.4602272727272727, "Indonesian,English,Spanish,Malay": 0.5511363636363636, "Indonesian,English,Spanish,Vietnamese": 0.5738636363636364, "Indonesian,English,Chinese,Malay": 0.4659090909090909, "Indonesian,English,Chinese,Vietnamese": 0.48295454545454547, "Indonesian,English,Malay,Vietnamese": 0.5454545454545454, "Indonesian,Filipino,Spanish,Chinese": 0.4431818181818182, "Indonesian,Filipino,Spanish,Malay": 0.5454545454545454, "Indonesian,Filipino,Spanish,Vietnamese": 0.5340909090909091, "Indonesian,Filipino,Chinese,Malay": 0.4715909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Malay,Vietnamese": 0.5511363636363636, "Indonesian,Spanish,Chinese,Malay": 0.44886363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.4431818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.5454545454545454, "Indonesian,Chinese,Malay,Vietnamese": 0.4659090909090909, "English,Filipino,Spanish,Chinese": 0.48863636363636365, "English,Filipino,Spanish,Malay": 0.5568181818181818, "English,Filipino,Spanish,Vietnamese": 0.5738636363636364, "English,Filipino,Chinese,Malay": 0.48863636363636365, "English,Filipino,Chinese,Vietnamese": 0.48863636363636365, "English,Filipino,Malay,Vietnamese": 0.5568181818181818, "English,Spanish,Chinese,Malay": 0.4602272727272727, "English,Spanish,Chinese,Vietnamese": 0.4943181818181818, "English,Spanish,Malay,Vietnamese": 0.5681818181818182, "English,Chinese,Malay,Vietnamese": 0.48295454545454547, "Filipino,Spanish,Chinese,Malay": 0.4659090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.45454545454545453, "Filipino,Spanish,Malay,Vietnamese": 0.5397727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.4659090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.44886363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.42045454545454547, "Indonesian,English,Filipino,Spanish,Malay": 0.5, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5113636363636364, "Indonesian,English,Filipino,Chinese,Malay": 0.4375, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4375, "Indonesian,English,Filipino,Malay,Vietnamese": 0.5056818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.42045454545454547, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4318181818181818, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5056818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4375, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4943181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.4375, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.5170454545454546, "English,Filipino,Chinese,Malay,Vietnamese": 0.44886363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.4375, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3977272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.39204545454545453, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4715909090909091, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.4090909090909091, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4147727272727273 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375 } }, "AC3_2": 0.5860106336131864, "AC3_3": 0.5368746071585395, "AC3_4": 0.5013618807436455, "AC3_5": 0.4735217048055425, "AC3_6": 0.44983055366842767, "AC3_7": 0.4279739776461651 }, "prompt_4": { "overall_acc": 0.49269480519480524, "language_acc": { "Indonesian": 0.48295454545454547, "English": 0.5113636363636364, "Filipino": 0.5, "Spanish": 0.48295454545454547, "Chinese": 0.5, "Malay": 0.5056818181818182, "Vietnamese": 0.4659090909090909 }, "consistency_score_2": 0.7058982683982685, "consistency_score_3": 0.5751623376623378, "consistency_score_4": 0.4956168831168831, "consistency_score_5": 0.439935064935065, "consistency_score_6": 0.39772727272727276, "consistency_score_7": 0.36363636363636365, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7272727272727273, "Indonesian,Filipino": 0.7215909090909091, "Indonesian,Spanish": 0.7443181818181818, "Indonesian,Chinese": 0.6590909090909091, "Indonesian,Malay": 0.7386363636363636, "Indonesian,Vietnamese": 0.7386363636363636, "English,Filipino": 0.7556818181818182, "English,Spanish": 0.7784090909090909, "English,Chinese": 0.6875, "English,Malay": 0.6647727272727273, "English,Vietnamese": 0.75, "Filipino,Spanish": 0.7329545454545454, "Filipino,Chinese": 0.6761363636363636, "Filipino,Malay": 0.6988636363636364, "Filipino,Vietnamese": 0.6875, "Spanish,Chinese": 0.6363636363636364, "Spanish,Malay": 0.7159090909090909, "Spanish,Vietnamese": 0.7386363636363636, "Chinese,Malay": 0.6193181818181818, "Chinese,Vietnamese": 0.6534090909090909, "Malay,Vietnamese": 0.6988636363636364 }, "3_combine": { "Indonesian,English,Filipino": 0.6079545454545454, "Indonesian,English,Spanish": 0.6306818181818182, "Indonesian,English,Chinese": 0.5454545454545454, "Indonesian,English,Malay": 0.5795454545454546, "Indonesian,English,Vietnamese": 0.625, "Indonesian,Filipino,Spanish": 0.6079545454545454, "Indonesian,Filipino,Chinese": 0.5511363636363636, "Indonesian,Filipino,Malay": 0.5965909090909091, "Indonesian,Filipino,Vietnamese": 0.6022727272727273, "Indonesian,Spanish,Chinese": 0.5284090909090909, "Indonesian,Spanish,Malay": 0.6136363636363636, "Indonesian,Spanish,Vietnamese": 0.6193181818181818, "Indonesian,Chinese,Malay": 0.5227272727272727, "Indonesian,Chinese,Vietnamese": 0.5454545454545454, "Indonesian,Malay,Vietnamese": 0.6079545454545454, "English,Filipino,Spanish": 0.6363636363636364, "English,Filipino,Chinese": 0.5795454545454546, "English,Filipino,Malay": 0.5738636363636364, "English,Filipino,Vietnamese": 0.6193181818181818, "English,Spanish,Chinese": 0.5681818181818182, "English,Spanish,Malay": 0.5909090909090909, "English,Spanish,Vietnamese": 0.6534090909090909, "English,Chinese,Malay": 0.5, "English,Chinese,Vietnamese": 0.5625, "English,Malay,Vietnamese": 0.5795454545454546, "Filipino,Spanish,Chinese": 0.5454545454545454, "Filipino,Spanish,Malay": 0.5909090909090909, "Filipino,Spanish,Vietnamese": 0.6022727272727273, "Filipino,Chinese,Malay": 0.5170454545454546, "Filipino,Chinese,Vietnamese": 0.5227272727272727, "Filipino,Malay,Vietnamese": 0.5738636363636364, "Spanish,Chinese,Malay": 0.4943181818181818, "Spanish,Chinese,Vietnamese": 0.5284090909090909, "Spanish,Malay,Vietnamese": 0.5965909090909091, "Chinese,Malay,Vietnamese": 0.5113636363636364 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.5511363636363636, "Indonesian,English,Filipino,Chinese": 0.4943181818181818, "Indonesian,English,Filipino,Malay": 0.5113636363636364, "Indonesian,English,Filipino,Vietnamese": 0.5511363636363636, "Indonesian,English,Spanish,Chinese": 0.48295454545454547, "Indonesian,English,Spanish,Malay": 0.5227272727272727, "Indonesian,English,Spanish,Vietnamese": 0.5625, "Indonesian,English,Chinese,Malay": 0.44886363636363635, "Indonesian,English,Chinese,Vietnamese": 0.5, "Indonesian,English,Malay,Vietnamese": 0.5113636363636364, "Indonesian,Filipino,Spanish,Chinese": 0.4715909090909091, "Indonesian,Filipino,Spanish,Malay": 0.5284090909090909, "Indonesian,Filipino,Spanish,Vietnamese": 0.5340909090909091, "Indonesian,Filipino,Chinese,Malay": 0.4602272727272727, "Indonesian,Filipino,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Chinese,Malay": 0.44886363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.4602272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.5284090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.4602272727272727, "English,Filipino,Spanish,Chinese": 0.5, "English,Filipino,Spanish,Malay": 0.5170454545454546, "English,Filipino,Spanish,Vietnamese": 0.5625, "English,Filipino,Chinese,Malay": 0.4659090909090909, "English,Filipino,Chinese,Vietnamese": 0.4943181818181818, "English,Filipino,Malay,Vietnamese": 0.5170454545454546, "English,Spanish,Chinese,Malay": 0.44886363636363635, "English,Spanish,Chinese,Vietnamese": 0.5, "English,Spanish,Malay,Vietnamese": 0.5397727272727273, "English,Chinese,Malay,Vietnamese": 0.4602272727272727, "Filipino,Spanish,Chinese,Malay": 0.45454545454545453, "Filipino,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.5170454545454546, "Filipino,Chinese,Malay,Vietnamese": 0.4431818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.4431818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4431818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.4715909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5056818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.42045454545454547, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.44886363636363635, "Indonesian,English,Filipino,Malay,Vietnamese": 0.4715909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.4090909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4431818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.4772727272727273, "Indonesian,English,Chinese,Malay,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4147727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4715909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.4090909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4034090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, "English,Filipino,Spanish,Chinese,Vietnamese": 0.44886363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "English,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.38636363636363635, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.39204545454545453, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.36363636363636365 } }, "AC3_2": 0.5803344228650118, "AC3_3": 0.5307442063367972, "AC3_4": 0.49415152434544735, "AC3_5": 0.46482260119573776, "AC3_6": 0.4401466809814796, "AC3_7": 0.4184403273962025 }, "prompt_5": { "overall_acc": 0.4886363636363637, "language_acc": { "Indonesian": 0.45454545454545453, "English": 0.5284090909090909, "Filipino": 0.48863636363636365, "Spanish": 0.48295454545454547, "Chinese": 0.5056818181818182, "Malay": 0.5056818181818182, "Vietnamese": 0.45454545454545453 }, "consistency_score_2": 0.6999458874458876, "consistency_score_3": 0.5681818181818182, "consistency_score_4": 0.48814935064935044, "consistency_score_5": 0.4320887445887445, "consistency_score_6": 0.3887987012987013, "consistency_score_7": 0.3522727272727273, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6761363636363636, "Indonesian,Filipino": 0.7272727272727273, "Indonesian,Spanish": 0.7159090909090909, "Indonesian,Chinese": 0.6136363636363636, "Indonesian,Malay": 0.7159090909090909, "Indonesian,Vietnamese": 0.7443181818181818, "English,Filipino": 0.75, "English,Spanish": 0.7613636363636364, "English,Chinese": 0.6647727272727273, "English,Malay": 0.7045454545454546, "English,Vietnamese": 0.7443181818181818, "Filipino,Spanish": 0.7386363636363636, "Filipino,Chinese": 0.6420454545454546, "Filipino,Malay": 0.7159090909090909, "Filipino,Vietnamese": 0.7045454545454546, "Spanish,Chinese": 0.6193181818181818, "Spanish,Malay": 0.7329545454545454, "Spanish,Vietnamese": 0.7443181818181818, "Chinese,Malay": 0.6136363636363636, "Chinese,Vietnamese": 0.6534090909090909, "Malay,Vietnamese": 0.7159090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.5909090909090909, "Indonesian,English,Spanish": 0.5909090909090909, "Indonesian,English,Chinese": 0.5, "Indonesian,English,Malay": 0.5681818181818182, "Indonesian,English,Vietnamese": 0.6022727272727273, "Indonesian,Filipino,Spanish": 0.5965909090909091, "Indonesian,Filipino,Chinese": 0.5056818181818182, "Indonesian,Filipino,Malay": 0.5909090909090909, "Indonesian,Filipino,Vietnamese": 0.6079545454545454, "Indonesian,Spanish,Chinese": 0.48863636363636365, "Indonesian,Spanish,Malay": 0.5965909090909091, "Indonesian,Spanish,Vietnamese": 0.6136363636363636, "Indonesian,Chinese,Malay": 0.4943181818181818, "Indonesian,Chinese,Vietnamese": 0.5284090909090909, "Indonesian,Malay,Vietnamese": 0.6136363636363636, "English,Filipino,Spanish": 0.6306818181818182, "English,Filipino,Chinese": 0.5511363636363636, "English,Filipino,Malay": 0.6022727272727273, "English,Filipino,Vietnamese": 0.6193181818181818, "English,Spanish,Chinese": 0.5397727272727273, "English,Spanish,Malay": 0.6136363636363636, "English,Spanish,Vietnamese": 0.6420454545454546, "English,Chinese,Malay": 0.5056818181818182, "English,Chinese,Vietnamese": 0.5568181818181818, "English,Malay,Vietnamese": 0.6079545454545454, "Filipino,Spanish,Chinese": 0.5227272727272727, "Filipino,Spanish,Malay": 0.6079545454545454, "Filipino,Spanish,Vietnamese": 0.6136363636363636, "Filipino,Chinese,Malay": 0.5113636363636364, "Filipino,Chinese,Vietnamese": 0.5227272727272727, "Filipino,Malay,Vietnamese": 0.5965909090909091, "Spanish,Chinese,Malay": 0.4943181818181818, "Spanish,Chinese,Vietnamese": 0.5284090909090909, "Spanish,Malay,Vietnamese": 0.6136363636363636, "Chinese,Malay,Vietnamese": 0.5170454545454546 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.5284090909090909, "Indonesian,English,Filipino,Chinese": 0.45454545454545453, "Indonesian,English,Filipino,Malay": 0.5, "Indonesian,English,Filipino,Vietnamese": 0.5397727272727273, "Indonesian,English,Spanish,Chinese": 0.4431818181818182, "Indonesian,English,Spanish,Malay": 0.5113636363636364, "Indonesian,English,Spanish,Vietnamese": 0.5397727272727273, "Indonesian,English,Chinese,Malay": 0.4318181818181818, "Indonesian,English,Chinese,Vietnamese": 0.4772727272727273, "Indonesian,English,Malay,Vietnamese": 0.5284090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.4431818181818182, "Indonesian,Filipino,Spanish,Malay": 0.5170454545454546, "Indonesian,Filipino,Spanish,Vietnamese": 0.5340909090909091, "Indonesian,Filipino,Chinese,Malay": 0.4318181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Malay,Vietnamese": 0.5284090909090909, "Indonesian,Spanish,Chinese,Malay": 0.42613636363636365, "Indonesian,Spanish,Chinese,Vietnamese": 0.4431818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.5340909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.44886363636363635, "English,Filipino,Spanish,Chinese": 0.4715909090909091, "English,Filipino,Spanish,Malay": 0.5397727272727273, "English,Filipino,Spanish,Vietnamese": 0.5568181818181818, "English,Filipino,Chinese,Malay": 0.45454545454545453, "English,Filipino,Chinese,Vietnamese": 0.48295454545454547, "English,Filipino,Malay,Vietnamese": 0.5397727272727273, "English,Spanish,Chinese,Malay": 0.4431818181818182, "English,Spanish,Chinese,Vietnamese": 0.48863636363636365, "English,Spanish,Malay,Vietnamese": 0.5511363636363636, "English,Chinese,Malay,Vietnamese": 0.4772727272727273, "Filipino,Spanish,Chinese,Malay": 0.44886363636363635, "Filipino,Spanish,Chinese,Vietnamese": 0.4715909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.5397727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.44886363636363635, "Spanish,Chinese,Malay,Vietnamese": 0.45454545454545453 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4090909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.4602272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.48863636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.39204545454545453, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4318181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.4772727272727273, "Indonesian,English,Spanish,Chinese,Malay": 0.39204545454545453, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.42045454545454547, "Indonesian,English,Spanish,Malay,Vietnamese": 0.48295454545454547, "Indonesian,English,Chinese,Malay,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.39204545454545453, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.3977272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.4090909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4375, "English,Filipino,Spanish,Malay,Vietnamese": 0.5, "English,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "English,Spanish,Chinese,Malay,Vietnamese": 0.4318181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.36363636363636365, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3806818181818182, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.3806818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3522727272727273 } }, "AC3_2": 0.575507522249451, "AC3_3": 0.5254154447205668, "AC3_4": 0.4883927356836032, "AC3_5": 0.4586260785439428, "AC3_6": 0.4330375914063701, "AC3_7": 0.4093980343493492 } }, "sg_eval": { "prompt_1": { "accuracy": 0.5048543689320388 }, "prompt_2": { "accuracy": 0.49514563106796117 }, "prompt_3": { "accuracy": 0.5242718446601942 }, "prompt_4": { "accuracy": 0.5145631067961165 }, "prompt_5": { "accuracy": 0.5339805825242718 } }, "cn_eval": { "prompt_1": { "accuracy": 0.3047619047619048 }, "prompt_2": { "accuracy": 0.3142857142857143 }, "prompt_3": { "accuracy": 0.29523809523809524 }, "prompt_4": { "accuracy": 0.3142857142857143 }, "prompt_5": { "accuracy": 0.3238095238095238 } }, "us_eval": { "prompt_1": { "accuracy": 0.2523364485981308 }, "prompt_2": { "accuracy": 0.2336448598130841 }, "prompt_3": { "accuracy": 0.24299065420560748 }, "prompt_4": { "accuracy": 0.2523364485981308 }, "prompt_5": { "accuracy": 0.24299065420560748 } }, "ph_eval": { "prompt_1": { "accuracy": 0.3, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.6, "history": 0.2, "literature": 0.0, "politics": 0.5, "culture": 0.3, "film": 0.1, "law": 0.3, "geography": 0.5 } }, "prompt_2": { "accuracy": 0.33, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.6, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.5, "culture": 0.2, "film": 0.1, "law": 0.3, "geography": 0.5 } }, "prompt_3": { "accuracy": 0.31, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.5, "history": 0.2, "literature": 0.2, "politics": 0.4, "culture": 0.4, "film": 0.1, "law": 0.3, "geography": 0.5 } }, "prompt_4": { "accuracy": 0.31, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.5, "history": 0.2, "literature": 0.2, "politics": 0.5, "culture": 0.3, "film": 0.1, "law": 0.3, "geography": 0.5 } }, "prompt_5": { "accuracy": 0.32, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.5, "history": 0.2, "literature": 0.3, "politics": 0.4, "culture": 0.3, "film": 0.1, "law": 0.3, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.1415726644994035 }, "prompt_2": { "bleu_score": 0.13732215511867826 }, "prompt_3": { "bleu_score": 0.1372456478046343 }, "prompt_4": { "bleu_score": 0.13881717658413173 }, "prompt_5": { "bleu_score": 0.1359329437121942 } }, "indommlu": { "prompt_1": { "accuracy": 0.5062420722344616, "category_acc": { "History": 0.3714859437751004, "Geography": 0.4326530612244898, "Lampungic": 0.4557823129251701, "Social science": 0.672787979966611, "Balinese": 0.4309978768577495, "Makassarese": 0.3924731182795699, "Banjarese": 0.4166666666666667, "Chemistry": 0.22773722627737225, "Biology": 0.3562130177514793, "Science": 0.5521155830753354, "Christian religion": 0.6069651741293532, "Art": 0.5640599001663894, "Islam religion": 0.6173541963015647, "Hindu religion": 0.5533333333333333, "Madurese": 0.3152542372881356, "Sport": 0.4594594594594595, "Indonesian language": 0.573785803237858, "Physics": 0.37373737373737376, "Minangkabau culture": 0.4020100502512563, "Dayak language": 0.3577981651376147, "Sociology": 0.4798387096774194, "Economy": 0.4733606557377049, "Sundanese": 0.6076058772687987, "Javanese": 0.5362903225806451, "Civic education": 0.5693848354792561 } }, "prompt_2": { "accuracy": 0.5037719473930169, "category_acc": { "History": 0.37751004016064255, "Geography": 0.4489795918367347, "Lampungic": 0.40816326530612246, "Social science": 0.671118530884808, "Balinese": 0.42250530785562634, "Makassarese": 0.3870967741935484, "Banjarese": 0.4166666666666667, "Chemistry": 0.22335766423357664, "Biology": 0.3562130177514793, "Science": 0.542827657378741, "Christian religion": 0.5870646766169154, "Art": 0.5490848585690515, "Islam religion": 0.6145092460881935, "Hindu religion": 0.54, "Madurese": 0.3050847457627119, "Sport": 0.46621621621621623, "Indonesian language": 0.5688044831880449, "Physics": 0.37777777777777777, "Minangkabau culture": 0.39195979899497485, "Dayak language": 0.3302752293577982, "Sociology": 0.48185483870967744, "Economy": 0.48565573770491804, "Sundanese": 0.6093344857389801, "Javanese": 0.5383064516129032, "Civic education": 0.575107296137339 } }, "prompt_3": { "accuracy": 0.5051739101408639, "category_acc": { "History": 0.3714859437751004, "Geography": 0.43673469387755104, "Lampungic": 0.41496598639455784, "Social science": 0.667779632721202, "Balinese": 0.4309978768577495, "Makassarese": 0.3978494623655914, "Banjarese": 0.4166666666666667, "Chemistry": 0.22773722627737225, "Biology": 0.35502958579881655, "Science": 0.543859649122807, "Christian religion": 0.6069651741293532, "Art": 0.5490848585690515, "Islam religion": 0.6173541963015647, "Hindu religion": 0.5533333333333333, "Madurese": 0.3050847457627119, "Sport": 0.43243243243243246, "Indonesian language": 0.5712951432129514, "Physics": 0.3717171717171717, "Minangkabau culture": 0.40703517587939697, "Dayak language": 0.3577981651376147, "Sociology": 0.4899193548387097, "Economy": 0.48155737704918034, "Sundanese": 0.6093344857389801, "Javanese": 0.5352822580645161, "Civic education": 0.5879828326180258 } }, "prompt_4": { "accuracy": 0.5062420722344616, "category_acc": { "History": 0.3855421686746988, "Geography": 0.4448979591836735, "Lampungic": 0.41496598639455784, "Social science": 0.674457429048414, "Balinese": 0.42250530785562634, "Makassarese": 0.3978494623655914, "Banjarese": 0.4027777777777778, "Chemistry": 0.22189781021897811, "Biology": 0.3668639053254438, "Science": 0.5469556243550051, "Christian religion": 0.5920398009950248, "Art": 0.5391014975041597, "Islam religion": 0.6187766714082503, "Hindu religion": 0.54, "Madurese": 0.33559322033898303, "Sport": 0.4594594594594595, "Indonesian language": 0.5669364881693649, "Physics": 0.3717171717171717, "Minangkabau culture": 0.39195979899497485, "Dayak language": 0.3394495412844037, "Sociology": 0.4899193548387097, "Economy": 0.4918032786885246, "Sundanese": 0.6093344857389801, "Javanese": 0.5433467741935484, "Civic education": 0.5894134477825465 } }, "prompt_5": { "accuracy": 0.5023699846451699, "category_acc": { "History": 0.3895582329317269, "Geography": 0.4489795918367347, "Lampungic": 0.38095238095238093, "Social science": 0.672787979966611, "Balinese": 0.4118895966029724, "Makassarese": 0.3870967741935484, "Banjarese": 0.4236111111111111, "Chemistry": 0.2321167883211679, "Biology": 0.35502958579881655, "Science": 0.5345717234262126, "Christian religion": 0.6019900497512438, "Art": 0.5440931780366056, "Islam religion": 0.604551920341394, "Hindu religion": 0.54, "Madurese": 0.3152542372881356, "Sport": 0.44594594594594594, "Indonesian language": 0.5647571606475716, "Physics": 0.37777777777777777, "Minangkabau culture": 0.4020100502512563, "Dayak language": 0.3486238532110092, "Sociology": 0.49798387096774194, "Economy": 0.47540983606557374, "Sundanese": 0.6101987899740708, "Javanese": 0.5282258064516129, "Civic education": 0.5822603719599427 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.3702499286280674 }, "prompt_2": { "bleu_score": 0.3725328734892377 }, "prompt_3": { "bleu_score": 0.3682673142036171 }, "prompt_4": { "bleu_score": 0.3729836664115557 }, "prompt_5": { "bleu_score": 0.3738645804241501 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.3331210760216344 }, "prompt_2": { "bleu_score": 0.3385870277660897 }, "prompt_3": { "bleu_score": 0.3340723812360414 }, "prompt_4": { "bleu_score": 0.34067568908971596 }, "prompt_5": { "bleu_score": 0.3404965901668535 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.2777190863401374 }, "prompt_2": { "bleu_score": 0.28465043692335784 }, "prompt_3": { "bleu_score": 0.27745720654550554 }, "prompt_4": { "bleu_score": 0.28738979676124116 }, "prompt_5": { "bleu_score": 0.28484775494547243 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.37879242173955846 }, "prompt_2": { "bleu_score": 0.3834367453020953 }, "prompt_3": { "bleu_score": 0.37836560188389307 }, "prompt_4": { "bleu_score": 0.38465772917800367 }, "prompt_5": { "bleu_score": 0.382863345203959 } }, "mmlu": { "prompt_1": { "accuracy": 0.4515752625437573 }, "prompt_2": { "accuracy": 0.4492415402567094 }, "prompt_3": { "accuracy": 0.44807467911318555 }, "prompt_4": { "accuracy": 0.4457409568261377 }, "prompt_5": { "accuracy": 0.45274212368728123 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.42710046478369684, "category_acc": { "high_school_european_history": 0.6646341463414634, "business_ethics": 0.5454545454545454, "clinical_knowledge": 0.4393939393939394, "medical_genetics": 0.3939393939393939, "high_school_us_history": 0.47783251231527096, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.6483050847457628, "virology": 0.3878787878787879, "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.26548672566371684, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.45307443365695793, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.3202846975088968, "philosophy": 0.44516129032258067, "professional_medicine": 0.4095940959409594, "nutrition": 0.4524590163934426, "global_facts": 0.29292929292929293, "machine_learning": 0.23423423423423423, "security_studies": 0.5204918032786885, "public_relations": 0.4954128440366973, "professional_psychology": 0.40098199672667756, "prehistory": 0.43034055727554177, "anatomy": 0.48507462686567165, "human_sexuality": 0.5, "college_medicine": 0.37790697674418605, "high_school_government_and_politics": 0.53125, "college_chemistry": 0.32323232323232326, "logical_fallacies": 0.5864197530864198, "high_school_geography": 0.5888324873096447, "elementary_mathematics": 0.3050397877984085, "human_aging": 0.4369369369369369, "college_mathematics": 0.35353535353535354, "high_school_psychology": 0.5735294117647058, "formal_logic": 0.32, "high_school_statistics": 0.33488372093023255, "international_law": 0.65, "high_school_mathematics": 0.26394052044609667, "high_school_computer_science": 0.42424242424242425, "conceptual_physics": 0.32905982905982906, "miscellaneous": 0.5332480818414322, "high_school_chemistry": 0.297029702970297, "marketing": 0.759656652360515, "professional_law": 0.3268101761252446, "management": 0.6372549019607843, "college_physics": 0.27722772277227725, "jurisprudence": 0.5887850467289719, "world_religions": 0.48823529411764705, "sociology": 0.605, "us_foreign_policy": 0.5858585858585859, "high_school_macroeconomics": 0.4473007712082262, "computer_security": 0.5454545454545454, "moral_scenarios": 0.27181208053691275, "moral_disputes": 0.4927536231884058, "electrical_engineering": 0.4583333333333333, "astronomy": 0.3841059602649007, "college_biology": 0.2937062937062937 } }, "prompt_2": { "accuracy": 0.421809081158384, "category_acc": { "high_school_european_history": 0.6646341463414634, "business_ethics": 0.5656565656565656, "clinical_knowledge": 0.4431818181818182, "medical_genetics": 0.43434343434343436, "high_school_us_history": 0.46798029556650245, "high_school_physics": 0.28, "high_school_world_history": 0.6313559322033898, "virology": 0.3939393939393939, "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.2831858407079646, "college_computer_science": 0.43434343434343436, "high_school_biology": 0.44660194174757284, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.3309608540925267, "philosophy": 0.43870967741935485, "professional_medicine": 0.3837638376383764, "nutrition": 0.4426229508196721, "global_facts": 0.3333333333333333, "machine_learning": 0.23423423423423423, "security_studies": 0.5163934426229508, "public_relations": 0.45871559633027525, "professional_psychology": 0.3911620294599018, "prehistory": 0.4179566563467492, "anatomy": 0.4925373134328358, "human_sexuality": 0.5307692307692308, "college_medicine": 0.36627906976744184, "high_school_government_and_politics": 0.5260416666666666, "college_chemistry": 0.3333333333333333, "logical_fallacies": 0.5925925925925926, "high_school_geography": 0.5786802030456852, "elementary_mathematics": 0.3050397877984085, "human_aging": 0.42792792792792794, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.5625, "formal_logic": 0.312, "high_school_statistics": 0.31627906976744186, "international_law": 0.65, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.45454545454545453, "conceptual_physics": 0.3418803418803419, "miscellaneous": 0.5281329923273658, "high_school_chemistry": 0.26732673267326734, "marketing": 0.7553648068669528, "professional_law": 0.32159165035877363, "management": 0.6176470588235294, "college_physics": 0.2871287128712871, "jurisprudence": 0.5420560747663551, "world_religions": 0.4823529411764706, "sociology": 0.59, "us_foreign_policy": 0.6060606060606061, "high_school_macroeconomics": 0.4190231362467866, "computer_security": 0.5353535353535354, "moral_scenarios": 0.24272930648769575, "moral_disputes": 0.5072463768115942, "electrical_engineering": 0.4583333333333333, "astronomy": 0.3841059602649007, "college_biology": 0.3356643356643357 } }, "prompt_3": { "accuracy": 0.4221666070790132, "category_acc": { "high_school_european_history": 0.6585365853658537, "business_ethics": 0.5252525252525253, "clinical_knowledge": 0.4393939393939394, "medical_genetics": 0.41414141414141414, "high_school_us_history": 0.4729064039408867, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.635593220338983, "virology": 0.3878787878787879, "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.2831858407079646, "college_computer_science": 0.42424242424242425, "high_school_biology": 0.4401294498381877, "abstract_algebra": 0.26262626262626265, "professional_accounting": 0.3416370106761566, "philosophy": 0.44516129032258067, "professional_medicine": 0.3874538745387454, "nutrition": 0.45901639344262296, "global_facts": 0.29292929292929293, "machine_learning": 0.23423423423423423, "security_studies": 0.5122950819672131, "public_relations": 0.48623853211009177, "professional_psychology": 0.39279869067103107, "prehistory": 0.4148606811145511, "anatomy": 0.5, "human_sexuality": 0.5230769230769231, "college_medicine": 0.38953488372093026, "high_school_government_and_politics": 0.5260416666666666, "college_chemistry": 0.31313131313131315, "logical_fallacies": 0.5802469135802469, "high_school_geography": 0.5634517766497462, "elementary_mathematics": 0.2917771883289125, "human_aging": 0.44144144144144143, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.5569852941176471, "formal_logic": 0.312, "high_school_statistics": 0.3023255813953488, "international_law": 0.6416666666666667, "high_school_mathematics": 0.2899628252788104, "high_school_computer_science": 0.41414141414141414, "conceptual_physics": 0.34615384615384615, "miscellaneous": 0.540920716112532, "high_school_chemistry": 0.26732673267326734, "marketing": 0.7467811158798283, "professional_law": 0.31637312459230266, "management": 0.6078431372549019, "college_physics": 0.2871287128712871, "jurisprudence": 0.5607476635514018, "world_religions": 0.47058823529411764, "sociology": 0.595, "us_foreign_policy": 0.5757575757575758, "high_school_macroeconomics": 0.43444730077120824, "computer_security": 0.5454545454545454, "moral_scenarios": 0.25727069351230425, "moral_disputes": 0.5043478260869565, "electrical_engineering": 0.4722222222222222, "astronomy": 0.40397350993377484, "college_biology": 0.3076923076923077 } }, "prompt_4": { "accuracy": 0.42481229889166966, "category_acc": { "high_school_european_history": 0.6585365853658537, "business_ethics": 0.5151515151515151, "clinical_knowledge": 0.4393939393939394, "medical_genetics": 0.41414141414141414, "high_school_us_history": 0.46798029556650245, "high_school_physics": 0.3, "high_school_world_history": 0.635593220338983, "virology": 0.4, "high_school_microeconomics": 0.4388185654008439, "econometrics": 0.2743362831858407, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.4336569579288026, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.3238434163701068, "philosophy": 0.43548387096774194, "professional_medicine": 0.3948339483394834, "nutrition": 0.46885245901639344, "global_facts": 0.31313131313131315, "machine_learning": 0.24324324324324326, "security_studies": 0.5081967213114754, "public_relations": 0.45871559633027525, "professional_psychology": 0.3993453355155483, "prehistory": 0.43653250773993807, "anatomy": 0.4701492537313433, "human_sexuality": 0.5384615384615384, "college_medicine": 0.38372093023255816, "high_school_government_and_politics": 0.5104166666666666, "college_chemistry": 0.3333333333333333, "logical_fallacies": 0.5864197530864198, "high_school_geography": 0.5685279187817259, "elementary_mathematics": 0.2917771883289125, "human_aging": 0.42342342342342343, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.5606617647058824, "formal_logic": 0.344, "high_school_statistics": 0.3209302325581395, "international_law": 0.6333333333333333, "high_school_mathematics": 0.2936802973977695, "high_school_computer_science": 0.46464646464646464, "conceptual_physics": 0.3547008547008547, "miscellaneous": 0.5319693094629157, "high_school_chemistry": 0.25742574257425743, "marketing": 0.759656652360515, "professional_law": 0.3242009132420091, "management": 0.6176470588235294, "college_physics": 0.297029702970297, "jurisprudence": 0.5700934579439252, "world_religions": 0.47058823529411764, "sociology": 0.61, "us_foreign_policy": 0.5858585858585859, "high_school_macroeconomics": 0.4370179948586118, "computer_security": 0.5353535353535354, "moral_scenarios": 0.2651006711409396, "moral_disputes": 0.4956521739130435, "electrical_engineering": 0.4583333333333333, "astronomy": 0.40397350993377484, "college_biology": 0.32867132867132864 } }, "prompt_5": { "accuracy": 0.42273864855202004, "category_acc": { "high_school_european_history": 0.6585365853658537, "business_ethics": 0.5757575757575758, "clinical_knowledge": 0.45454545454545453, "medical_genetics": 0.46464646464646464, "high_school_us_history": 0.45320197044334976, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.6271186440677966, "virology": 0.3878787878787879, "high_school_microeconomics": 0.4388185654008439, "econometrics": 0.2743362831858407, "college_computer_science": 0.4444444444444444, "high_school_biology": 0.44983818770226536, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.33807829181494664, "philosophy": 0.43870967741935485, "professional_medicine": 0.3948339483394834, "nutrition": 0.4459016393442623, "global_facts": 0.30303030303030304, "machine_learning": 0.25225225225225223, "security_studies": 0.5122950819672131, "public_relations": 0.45871559633027525, "professional_psychology": 0.3895253682487725, "prehistory": 0.4086687306501548, "anatomy": 0.5149253731343284, "human_sexuality": 0.5230769230769231, "college_medicine": 0.3953488372093023, "high_school_government_and_politics": 0.5208333333333334, "college_chemistry": 0.32323232323232326, "logical_fallacies": 0.5802469135802469, "high_school_geography": 0.5685279187817259, "elementary_mathematics": 0.30238726790450926, "human_aging": 0.43243243243243246, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.5625, "formal_logic": 0.336, "high_school_statistics": 0.3209302325581395, "international_law": 0.6166666666666667, "high_school_mathematics": 0.26765799256505574, "high_school_computer_science": 0.4444444444444444, "conceptual_physics": 0.36752136752136755, "miscellaneous": 0.5358056265984654, "high_school_chemistry": 0.28217821782178215, "marketing": 0.759656652360515, "professional_law": 0.31833007175472927, "management": 0.6176470588235294, "college_physics": 0.2871287128712871, "jurisprudence": 0.5420560747663551, "world_religions": 0.4764705882352941, "sociology": 0.605, "us_foreign_policy": 0.5656565656565656, "high_school_macroeconomics": 0.4267352185089974, "computer_security": 0.5454545454545454, "moral_scenarios": 0.24049217002237136, "moral_disputes": 0.5072463768115942, "electrical_engineering": 0.4722222222222222, "astronomy": 0.37748344370860926, "college_biology": 0.3356643356643357 } } }, "c_eval": { "prompt_1": { "accuracy": 0.4078751857355126 }, "prompt_2": { "accuracy": 0.4078751857355126 }, "prompt_3": { "accuracy": 0.4034175334323923 }, "prompt_4": { "accuracy": 0.4063893016344725 }, "prompt_5": { "accuracy": 0.4086181277860327 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.3916562889165629, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.375, "computer_architecture": 0.4230769230769231, "college_programming": 0.2857142857142857, "college_physics": 0.20833333333333334, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.4782608695652174, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.5, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.4166666666666667, "business_administration": 0.39473684210526316, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.4827586206896552, "education_science": 0.5588235294117647, "teacher_qualification": 0.5918367346938775, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.5, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.17647058823529413, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.7083333333333334, "logic": 0.4444444444444444, "law": 0.3103448275862069, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.39473684210526316, "professional_tour_guide": 0.5, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.48, "middle_school_history": 0.37037037037037035, "civil_servant": 0.38461538461538464, "sports_science": 0.4166666666666667, "plant_protection": 0.4074074074074074, "basic_medicine": 0.375, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.47058823529411764, "accountant": 0.3148148148148148, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.2962962962962963, "physician": 0.3888888888888889 } }, "prompt_2": { "accuracy": 0.3891656288916563, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.3333333333333333, "computer_architecture": 0.46153846153846156, "college_programming": 0.3333333333333333, "college_physics": 0.25, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.25, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.5, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.4166666666666667, "business_administration": 0.39473684210526316, "marxism": 0.5, "mao_zedong_thought": 0.5517241379310345, "education_science": 0.5588235294117647, "teacher_qualification": 0.5714285714285714, "high_school_politics": 0.25, "high_school_geography": 0.5, "middle_school_politics": 0.6153846153846154, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.48148148148148145, "law": 0.3103448275862069, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.5, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.48, "middle_school_history": 0.4074074074074074, "civil_servant": 0.40384615384615385, "sports_science": 0.4583333333333333, "plant_protection": 0.4074074074074074, "basic_medicine": 0.375, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.49019607843137253, "accountant": 0.35185185185185186, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.25925925925925924, "physician": 0.3888888888888889 } }, "prompt_3": { "accuracy": 0.3953922789539228, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.375, "computer_architecture": 0.38461538461538464, "college_programming": 0.30952380952380953, "college_physics": 0.2916666666666667, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.16666666666666666, "probability_and_statistics": 0.4782608695652174, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.3333333333333333, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.16666666666666666, "high_school_biology": 0.25, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.5, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.4166666666666667, "business_administration": 0.42105263157894735, "marxism": 0.5, "mao_zedong_thought": 0.4827586206896552, "education_science": 0.5588235294117647, "teacher_qualification": 0.5918367346938775, "high_school_politics": 0.3333333333333333, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.75, "logic": 0.48148148148148145, "law": 0.3103448275862069, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.3157894736842105, "professional_tour_guide": 0.5294117647058824, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.52, "middle_school_history": 0.4074074074074074, "civil_servant": 0.4230769230769231, "sports_science": 0.4166666666666667, "plant_protection": 0.4444444444444444, "basic_medicine": 0.375, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.5098039215686274, "accountant": 0.3148148148148148, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.3888888888888889, "tax_accountant": 0.2962962962962963, "physician": 0.4074074074074074 } }, "prompt_4": { "accuracy": 0.3854296388542964, "category_acc": { "computer_network": 0.375, "operating_system": 0.3333333333333333, "computer_architecture": 0.4230769230769231, "college_programming": 0.3333333333333333, "college_physics": 0.25, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.16666666666666666, "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.4482758620689655, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.16666666666666666, "high_school_biology": 0.25, "middle_school_mathematics": 0.25, "middle_school_biology": 0.5, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.43333333333333335, "business_administration": 0.3684210526315789, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.4827586206896552, "education_science": 0.5294117647058824, "teacher_qualification": 0.5714285714285714, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.6538461538461539, "middle_school_geography": 0.17647058823529413, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.48148148148148145, "law": 0.3103448275862069, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.5, "legal_professional": 0.35714285714285715, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.44, "middle_school_history": 0.4074074074074074, "civil_servant": 0.40384615384615385, "sports_science": 0.375, "plant_protection": 0.4074074074074074, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.5098039215686274, "accountant": 0.3148148148148148, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.24074074074074073, "physician": 0.37037037037037035 } }, "prompt_5": { "accuracy": 0.3916562889165629, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.3333333333333333, "computer_architecture": 0.38461538461538464, "college_programming": 0.2857142857142857, "college_physics": 0.2916666666666667, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.25, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.5, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.43333333333333335, "business_administration": 0.42105263157894735, "marxism": 0.5, "mao_zedong_thought": 0.5862068965517241, "education_science": 0.5882352941176471, "teacher_qualification": 0.5918367346938775, "high_school_politics": 0.4583333333333333, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.42857142857142855, "ideological_and_moral_cultivation": 0.625, "logic": 0.48148148148148145, "law": 0.3103448275862069, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.25, "high_school_history": 0.48, "middle_school_history": 0.4074074074074074, "civil_servant": 0.38461538461538464, "sports_science": 0.375, "plant_protection": 0.4074074074074074, "basic_medicine": 0.375, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.47058823529411764, "accountant": 0.3148148148148148, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.2962962962962963, "physician": 0.4074074074074074 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.4121863799283154 }, "prompt_2": { "accuracy": 0.40860215053763443 }, "prompt_3": { "accuracy": 0.3978494623655914 }, "prompt_4": { "accuracy": 0.3978494623655914 }, "prompt_5": { "accuracy": 0.4265232974910394 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.4134001036090485, "category_acc": { "agronomy": 0.40828402366863903, "anatomy": 0.32432432432432434, "ancient_chinese": 0.23170731707317074, "arts": 0.43125, "astronomy": 0.2, "business_ethics": 0.46411483253588515, "chinese_civil_service_exam": 0.425, "chinese_driving_rule": 0.6564885496183206, "chinese_food_culture": 0.36764705882352944, "chinese_foreign_policy": 0.45794392523364486, "chinese_history": 0.47678018575851394, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.5083798882681564, "clinical_knowledge": 0.3206751054852321, "college_actuarial_science": 0.27358490566037735, "college_education": 0.5700934579439252, "college_engineering_hydrology": 0.33962264150943394, "college_law": 0.32407407407407407, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.4056603773584906, "college_medicine": 0.31868131868131866, "computer_science": 0.44607843137254904, "computer_security": 0.4444444444444444, "conceptual_physics": 0.3197278911564626, "construction_project_management": 0.381294964028777, "economics": 0.4716981132075472, "education": 0.4785276073619632, "electrical_engineering": 0.38953488372093026, "elementary_chinese": 0.3134920634920635, "elementary_commonsense": 0.398989898989899, "elementary_information_and_technology": 0.6050420168067226, "elementary_mathematics": 0.30434782608695654, "ethnology": 0.4074074074074074, "food_science": 0.46153846153846156, "genetics": 0.2840909090909091, "global_facts": 0.35570469798657717, "high_school_biology": 0.3076923076923077, "high_school_chemistry": 0.32575757575757575, "high_school_geography": 0.3813559322033898, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.40559440559440557, "human_sexuality": 0.4444444444444444, "international_law": 0.43243243243243246, "journalism": 0.5232558139534884, "jurisprudence": 0.4768856447688564, "legal_and_moral_basis": 0.7850467289719626, "logical": 0.3983739837398374, "machine_learning": 0.36065573770491804, "management": 0.5, "marketing": 0.5444444444444444, "marxist_theory": 0.5238095238095238, "modern_chinese": 0.3103448275862069, "nutrition": 0.45517241379310347, "philosophy": 0.4095238095238095, "professional_accounting": 0.45714285714285713, "professional_law": 0.3459715639810427, "professional_medicine": 0.2898936170212766, "professional_psychology": 0.5129310344827587, "public_relations": 0.5114942528735632, "security_study": 0.5481481481481482, "sociology": 0.43805309734513276, "sports_science": 0.49696969696969695, "traditional_chinese_medicine": 0.32972972972972975, "virology": 0.4260355029585799, "world_history": 0.4409937888198758, "world_religions": 0.41875 } }, "prompt_2": { "accuracy": 0.41063719564841994, "category_acc": { "agronomy": 0.39644970414201186, "anatomy": 0.31756756756756754, "ancient_chinese": 0.24390243902439024, "arts": 0.45625, "astronomy": 0.21212121212121213, "business_ethics": 0.4688995215311005, "chinese_civil_service_exam": 0.425, "chinese_driving_rule": 0.648854961832061, "chinese_food_culture": 0.36764705882352944, "chinese_foreign_policy": 0.45794392523364486, "chinese_history": 0.47368421052631576, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.5195530726256983, "clinical_knowledge": 0.32489451476793246, "college_actuarial_science": 0.29245283018867924, "college_education": 0.5607476635514018, "college_engineering_hydrology": 0.330188679245283, "college_law": 0.3148148148148148, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.3867924528301887, "college_medicine": 0.3076923076923077, "computer_science": 0.4362745098039216, "computer_security": 0.43859649122807015, "conceptual_physics": 0.32653061224489793, "construction_project_management": 0.3381294964028777, "economics": 0.46540880503144655, "education": 0.49693251533742333, "electrical_engineering": 0.38372093023255816, "elementary_chinese": 0.2976190476190476, "elementary_commonsense": 0.3838383838383838, "elementary_information_and_technology": 0.592436974789916, "elementary_mathematics": 0.29130434782608694, "ethnology": 0.4074074074074074, "food_science": 0.48951048951048953, "genetics": 0.26136363636363635, "global_facts": 0.33557046979865773, "high_school_biology": 0.3254437869822485, "high_school_chemistry": 0.32575757575757575, "high_school_geography": 0.3813559322033898, "high_school_mathematics": 0.21341463414634146, "high_school_physics": 0.34545454545454546, "high_school_politics": 0.3916083916083916, "human_sexuality": 0.4603174603174603, "international_law": 0.41621621621621624, "journalism": 0.5348837209302325, "jurisprudence": 0.4768856447688564, "legal_and_moral_basis": 0.7523364485981309, "logical": 0.3821138211382114, "machine_learning": 0.3442622950819672, "management": 0.4857142857142857, "marketing": 0.5388888888888889, "marxist_theory": 0.5343915343915344, "modern_chinese": 0.3017241379310345, "nutrition": 0.4482758620689655, "philosophy": 0.4095238095238095, "professional_accounting": 0.44, "professional_law": 0.36018957345971564, "professional_medicine": 0.30319148936170215, "professional_psychology": 0.5129310344827587, "public_relations": 0.5172413793103449, "security_study": 0.5407407407407407, "sociology": 0.43805309734513276, "sports_science": 0.49696969696969695, "traditional_chinese_medicine": 0.32432432432432434, "virology": 0.4319526627218935, "world_history": 0.4409937888198758, "world_religions": 0.40625 } }, "prompt_3": { "accuracy": 0.41158694525988604, "category_acc": { "agronomy": 0.42011834319526625, "anatomy": 0.2972972972972973, "ancient_chinese": 0.2682926829268293, "arts": 0.43125, "astronomy": 0.20606060606060606, "business_ethics": 0.45933014354066987, "chinese_civil_service_exam": 0.41875, "chinese_driving_rule": 0.648854961832061, "chinese_food_culture": 0.39705882352941174, "chinese_foreign_policy": 0.4485981308411215, "chinese_history": 0.47058823529411764, "chinese_literature": 0.3235294117647059, "chinese_teacher_qualification": 0.5139664804469274, "clinical_knowledge": 0.3080168776371308, "college_actuarial_science": 0.29245283018867924, "college_education": 0.5607476635514018, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.3333333333333333, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.3867924528301887, "college_medicine": 0.31135531135531136, "computer_science": 0.43137254901960786, "computer_security": 0.4327485380116959, "conceptual_physics": 0.3333333333333333, "construction_project_management": 0.35251798561151076, "economics": 0.46540880503144655, "education": 0.4723926380368098, "electrical_engineering": 0.37790697674418605, "elementary_chinese": 0.31746031746031744, "elementary_commonsense": 0.3686868686868687, "elementary_information_and_technology": 0.6176470588235294, "elementary_mathematics": 0.32608695652173914, "ethnology": 0.4222222222222222, "food_science": 0.46153846153846156, "genetics": 0.26704545454545453, "global_facts": 0.37583892617449666, "high_school_biology": 0.31952662721893493, "high_school_chemistry": 0.32575757575757575, "high_school_geography": 0.3898305084745763, "high_school_mathematics": 0.23170731707317074, "high_school_physics": 0.35454545454545455, "high_school_politics": 0.40559440559440557, "human_sexuality": 0.4444444444444444, "international_law": 0.41621621621621624, "journalism": 0.5523255813953488, "jurisprudence": 0.48418491484184917, "legal_and_moral_basis": 0.7523364485981309, "logical": 0.3821138211382114, "machine_learning": 0.3360655737704918, "management": 0.47619047619047616, "marketing": 0.5333333333333333, "marxist_theory": 0.5238095238095238, "modern_chinese": 0.31896551724137934, "nutrition": 0.4689655172413793, "philosophy": 0.3904761904761905, "professional_accounting": 0.44571428571428573, "professional_law": 0.33175355450236965, "professional_medicine": 0.30319148936170215, "professional_psychology": 0.5172413793103449, "public_relations": 0.5057471264367817, "security_study": 0.562962962962963, "sociology": 0.4557522123893805, "sports_science": 0.49696969696969695, "traditional_chinese_medicine": 0.3027027027027027, "virology": 0.4319526627218935, "world_history": 0.40993788819875776, "world_religions": 0.39375 } }, "prompt_4": { "accuracy": 0.41201864962873425, "category_acc": { "agronomy": 0.41420118343195267, "anatomy": 0.31756756756756754, "ancient_chinese": 0.23170731707317074, "arts": 0.425, "astronomy": 0.2, "business_ethics": 0.45933014354066987, "chinese_civil_service_exam": 0.41875, "chinese_driving_rule": 0.6259541984732825, "chinese_food_culture": 0.375, "chinese_foreign_policy": 0.4766355140186916, "chinese_history": 0.48606811145510836, "chinese_literature": 0.29901960784313725, "chinese_teacher_qualification": 0.5139664804469274, "clinical_knowledge": 0.31223628691983124, "college_actuarial_science": 0.29245283018867924, "college_education": 0.5607476635514018, "college_engineering_hydrology": 0.33962264150943394, "college_law": 0.3333333333333333, "college_mathematics": 0.18095238095238095, "college_medical_statistics": 0.3867924528301887, "college_medicine": 0.30036630036630035, "computer_science": 0.45588235294117646, "computer_security": 0.4327485380116959, "conceptual_physics": 0.3197278911564626, "construction_project_management": 0.34532374100719426, "economics": 0.4968553459119497, "education": 0.4785276073619632, "electrical_engineering": 0.38372093023255816, "elementary_chinese": 0.2976190476190476, "elementary_commonsense": 0.3838383838383838, "elementary_information_and_technology": 0.6050420168067226, "elementary_mathematics": 0.3, "ethnology": 0.4148148148148148, "food_science": 0.46153846153846156, "genetics": 0.30113636363636365, "global_facts": 0.348993288590604, "high_school_biology": 0.31952662721893493, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.3983050847457627, "high_school_mathematics": 0.24390243902439024, "high_school_physics": 0.35454545454545455, "high_school_politics": 0.4125874125874126, "human_sexuality": 0.4603174603174603, "international_law": 0.42702702702702705, "journalism": 0.5058139534883721, "jurisprudence": 0.48418491484184917, "legal_and_moral_basis": 0.7757009345794392, "logical": 0.3902439024390244, "machine_learning": 0.3524590163934426, "management": 0.5, "marketing": 0.5444444444444444, "marxist_theory": 0.5079365079365079, "modern_chinese": 0.3103448275862069, "nutrition": 0.4482758620689655, "philosophy": 0.3619047619047619, "professional_accounting": 0.4514285714285714, "professional_law": 0.3459715639810427, "professional_medicine": 0.30851063829787234, "professional_psychology": 0.521551724137931, "public_relations": 0.5287356321839081, "security_study": 0.5481481481481482, "sociology": 0.4469026548672566, "sports_science": 0.5333333333333333, "traditional_chinese_medicine": 0.32972972972972975, "virology": 0.40236686390532544, "world_history": 0.4161490683229814, "world_religions": 0.41875 } }, "prompt_5": { "accuracy": 0.4110689000172682, "category_acc": { "agronomy": 0.3905325443786982, "anatomy": 0.30405405405405406, "ancient_chinese": 0.25, "arts": 0.45, "astronomy": 0.2, "business_ethics": 0.47368421052631576, "chinese_civil_service_exam": 0.41875, "chinese_driving_rule": 0.6564885496183206, "chinese_food_culture": 0.3602941176470588, "chinese_foreign_policy": 0.4672897196261682, "chinese_history": 0.47987616099071206, "chinese_literature": 0.3088235294117647, "chinese_teacher_qualification": 0.4972067039106145, "clinical_knowledge": 0.31223628691983124, "college_actuarial_science": 0.27358490566037735, "college_education": 0.5420560747663551, "college_engineering_hydrology": 0.3113207547169811, "college_law": 0.32407407407407407, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.39622641509433965, "college_medicine": 0.31868131868131866, "computer_science": 0.4411764705882353, "computer_security": 0.42105263157894735, "conceptual_physics": 0.3401360544217687, "construction_project_management": 0.3381294964028777, "economics": 0.48427672955974843, "education": 0.48466257668711654, "electrical_engineering": 0.38953488372093026, "elementary_chinese": 0.30158730158730157, "elementary_commonsense": 0.3939393939393939, "elementary_information_and_technology": 0.6008403361344538, "elementary_mathematics": 0.3130434782608696, "ethnology": 0.4, "food_science": 0.4755244755244755, "genetics": 0.2556818181818182, "global_facts": 0.33557046979865773, "high_school_biology": 0.3136094674556213, "high_school_chemistry": 0.3484848484848485, "high_school_geography": 0.4067796610169492, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.3986013986013986, "human_sexuality": 0.4603174603174603, "international_law": 0.42162162162162165, "journalism": 0.5174418604651163, "jurisprudence": 0.48905109489051096, "legal_and_moral_basis": 0.7663551401869159, "logical": 0.3821138211382114, "machine_learning": 0.36885245901639346, "management": 0.49047619047619045, "marketing": 0.5388888888888889, "marxist_theory": 0.5291005291005291, "modern_chinese": 0.3017241379310345, "nutrition": 0.46206896551724136, "philosophy": 0.4, "professional_accounting": 0.44571428571428573, "professional_law": 0.3412322274881517, "professional_medicine": 0.30319148936170215, "professional_psychology": 0.5086206896551724, "public_relations": 0.5, "security_study": 0.5185185185185185, "sociology": 0.4336283185840708, "sports_science": 0.509090909090909, "traditional_chinese_medicine": 0.31351351351351353, "virology": 0.41420118343195267, "world_history": 0.4409937888198758, "world_religions": 0.41875 } } }, "zbench": { "prompt_1": { "accuracy": 0.3333333333333333 }, "prompt_2": { "accuracy": 0.36363636363636365 }, "prompt_3": { "accuracy": 0.36363636363636365 }, "prompt_4": { "accuracy": 0.3333333333333333 }, "prompt_5": { "accuracy": 0.36363636363636365 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6227272727272727 }, "prompt_2": { "accuracy": 0.6136363636363636 }, "prompt_3": { "accuracy": 0.5977272727272728 }, "prompt_4": { "accuracy": 0.6136363636363636 }, "prompt_5": { "accuracy": 0.6113636363636363 } }, "ocnli": { "prompt_1": { "accuracy": 0.3925423728813559 }, "prompt_2": { "accuracy": 0.3959322033898305 }, "prompt_3": { "accuracy": 0.3752542372881356 }, "prompt_4": { "accuracy": 0.4152542372881356 }, "prompt_5": { "accuracy": 0.4542372881355932 } }, "c3": { "prompt_1": { "accuracy": 0.8702318623784593 }, "prompt_2": { "accuracy": 0.8709798055347794 }, "prompt_3": { "accuracy": 0.8739715781600599 }, "prompt_4": { "accuracy": 0.868362004487659 }, "prompt_5": { "accuracy": 0.8713537771129394 } }, "dream": { "prompt_1": { "accuracy": 0.8907398334149926 }, "prompt_2": { "accuracy": 0.8853503184713376 }, "prompt_3": { "accuracy": 0.8873101420872122 }, "prompt_4": { "accuracy": 0.8863302302792748 }, "prompt_5": { "accuracy": 0.8873101420872122 } }, "samsum": { "prompt_1": { "rouge1": 0.41259329359093655, "rouge2": 0.2115510244457223, "rougeL": 0.3530618026494755, "avg_rouge": 0.3257353735620448 }, "prompt_2": { "rouge1": 0.41707374512390494, "rouge2": 0.2124796660781807, "rougeL": 0.354080814542538, "avg_rouge": 0.32787807524820783 }, "prompt_3": { "rouge1": 0.41321357066826325, "rouge2": 0.20913215733645862, "rougeL": 0.3501808312576957, "avg_rouge": 0.3241755197541392 }, "prompt_4": { "rouge1": 0.41878254040446516, "rouge2": 0.2122266216884994, "rougeL": 0.35334307500453, "avg_rouge": 0.32811741236583153 }, "prompt_5": { "rouge1": 0.41256561894917226, "rouge2": 0.2101843136359623, "rougeL": 0.34918946838957315, "avg_rouge": 0.3239798003249026 } }, "dialogsum": { "prompt_1": { "rouge1": 0.19650427568779455, "rouge2": 0.05364004839716688, "rougeL": 0.14780535686319093, "avg_rouge": 0.13264989364938415 }, "prompt_2": { "rouge1": 0.1973451031462774, "rouge2": 0.05506989252399569, "rougeL": 0.14860981077086455, "avg_rouge": 0.1336749354803792 }, "prompt_3": { "rouge1": 0.20097510069358493, "rouge2": 0.06074357546102678, "rougeL": 0.15732985995613014, "avg_rouge": 0.13968284537024728 }, "prompt_4": { "rouge1": 0.19718866521664802, "rouge2": 0.055280092355201836, "rougeL": 0.14855552896452479, "avg_rouge": 0.13367476217879154 }, "prompt_5": { "rouge1": 0.2004555565400454, "rouge2": 0.05905148166979365, "rougeL": 0.15259325268376314, "avg_rouge": 0.13736676363120073 } }, "sst2": { "prompt_1": { "accuracy": 0.948394495412844 }, "prompt_2": { "accuracy": 0.9461009174311926 }, "prompt_3": { "accuracy": 0.9461009174311926 }, "prompt_4": { "accuracy": 0.9472477064220184 }, "prompt_5": { "accuracy": 0.9243119266055045 } }, "cola": { "prompt_1": { "accuracy": 0.5656759348034516 }, "prompt_2": { "accuracy": 0.573346116970278 }, "prompt_3": { "accuracy": 0.5695110258868649 }, "prompt_4": { "accuracy": 0.573346116970278 }, "prompt_5": { "accuracy": 0.5043144774688398 } }, "qqp": { "prompt_1": { "accuracy": 0.6895 }, "prompt_2": { "accuracy": 0.5115 }, "prompt_3": { "accuracy": 0.771 }, "prompt_4": { "accuracy": 0.635 }, "prompt_5": { "accuracy": 0.5615 } }, "mnli": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.7 } }, "qnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.5 } }, "wnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.6 } }, "rte": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.8 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.4 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "seallm_7b_v2": { "model_size": "7B", "model_link": "https://huggingface.co/SeaLLMs/SeaLLM-7B-v2", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9228991596638655, "language_acc": { "Spanish": 0.9285714285714286, "English": 0.9436974789915966, "Chinese": 0.9058823529411765, "Vietnamese": 0.9134453781512605 }, "consistency_score_2": 0.911624649859944, "consistency_score_3": 0.8710084033613446, "consistency_score_4": 0.8420168067226891, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9428571428571428, "Spanish,Chinese": 0.8991596638655462, "Spanish,Vietnamese": 0.9126050420168067, "English,Chinese": 0.9092436974789916, "English,Vietnamese": 0.9142857142857143, "Chinese,Vietnamese": 0.8915966386554622 }, "3_combine": { "Spanish,English,Chinese": 0.8781512605042017, "Spanish,English,Vietnamese": 0.8873949579831932, "Spanish,Chinese,Vietnamese": 0.8563025210084033, "English,Chinese,Vietnamese": 0.8621848739495799 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8420168067226891 } }, "AC3_2": 0.9172272595983739, "AC3_3": 0.8962032827621712, "AC3_4": 0.8806046498567287 }, "prompt_2": { "overall_acc": 0.915126050420168, "language_acc": { "Spanish": 0.9226890756302522, "English": 0.938655462184874, "Chinese": 0.9033613445378151, "Vietnamese": 0.8957983193277311 }, "consistency_score_2": 0.8943977591036415, "consistency_score_3": 0.8453781512605042, "consistency_score_4": 0.8058823529411765, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9310924369747899, "Spanish,Chinese": 0.8899159663865546, "Spanish,Vietnamese": 0.8789915966386554, "English,Chinese": 0.9092436974789916, "English,Vietnamese": 0.8949579831932774, "Chinese,Vietnamese": 0.8621848739495799 }, "3_combine": { "Spanish,English,Chinese": 0.8697478991596639, "Spanish,English,Vietnamese": 0.853781512605042, "Spanish,Chinese,Vietnamese": 0.8201680672268907, "English,Chinese,Vietnamese": 0.8378151260504202 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8058823529411765 } }, "AC3_2": 0.9046431823003461, "AC3_3": 0.8788704598285406, "AC3_4": 0.8570369944354955 }, "prompt_3": { "overall_acc": 0.9063025210084034, "language_acc": { "Spanish": 0.9117647058823529, "English": 0.938655462184874, "Chinese": 0.9033613445378151, "Vietnamese": 0.8714285714285714 }, "consistency_score_2": 0.8820728291316526, "consistency_score_3": 0.8294117647058823, "consistency_score_4": 0.7899159663865546, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9159663865546218, "Spanish,Chinese": 0.8815126050420168, "Spanish,Vietnamese": 0.8613445378151261, "English,Chinese": 0.9033613445378151, "English,Vietnamese": 0.8764705882352941, "Chinese,Vietnamese": 0.853781512605042 }, "3_combine": { "Spanish,English,Chinese": 0.8571428571428571, "Spanish,English,Vietnamese": 0.83109243697479, "Spanish,Chinese,Vietnamese": 0.807563025210084, "English,Chinese,Vietnamese": 0.8218487394957983 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.7899159663865546 } }, "AC3_2": 0.8940235377855366, "AC3_3": 0.866154043266771, "AC3_4": 0.8441162939783743 }, "prompt_4": { "overall_acc": 0.9163865546218487, "language_acc": { "Spanish": 0.9260504201680673, "English": 0.9361344537815126, "Chinese": 0.9025210084033614, "Vietnamese": 0.9008403361344538 }, "consistency_score_2": 0.9007002801120448, "consistency_score_3": 0.8552521008403362, "consistency_score_4": 0.8243697478991596, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9294117647058824, "Spanish,Chinese": 0.8941176470588236, "Spanish,Vietnamese": 0.8941176470588236, "English,Chinese": 0.9025210084033614, "English,Vietnamese": 0.9050420168067227, "Chinese,Vietnamese": 0.8789915966386554 }, "3_combine": { "Spanish,English,Chinese": 0.8680672268907563, "Spanish,English,Vietnamese": 0.8663865546218488, "Spanish,Chinese,Vietnamese": 0.8378151260504202, "English,Chinese,Vietnamese": 0.8487394957983193 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8243697478991596 } }, "AC3_2": 0.9084757102587919, "AC3_3": 0.8847645354334098, "AC3_4": 0.8679461356817676 }, "prompt_5": { "overall_acc": 0.9195378151260504, "language_acc": { "Spanish": 0.9226890756302522, "English": 0.9420168067226891, "Chinese": 0.915126050420168, "Vietnamese": 0.8983193277310925 }, "consistency_score_2": 0.9050420168067227, "consistency_score_3": 0.8621848739495799, "consistency_score_4": 0.8327731092436975, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9327731092436975, "Spanish,Chinese": 0.9058823529411765, "Spanish,Vietnamese": 0.8890756302521008, "English,Chinese": 0.9159663865546218, "English,Vietnamese": 0.9, "Chinese,Vietnamese": 0.8865546218487395 }, "3_combine": { "Spanish,English,Chinese": 0.8815126050420168, "Spanish,English,Vietnamese": 0.8638655462184874, "Spanish,Chinese,Vietnamese": 0.8478991596638655, "English,Chinese,Vietnamese": 0.8554621848739495 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8327731092436975 } }, "AC3_2": 0.91223233329785, "AC3_3": 0.8899382603621049, "AC3_4": 0.8740074089321783 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.6085714285714285, "language_acc": { "Filipino": 0.5733333333333334, "Vietnamese": 0.56, "Chinese": 0.5733333333333334, "Spanish": 0.6533333333333333, "Malay": 0.54, "Indonesian": 0.6266666666666667, "English": 0.7333333333333333 }, "consistency_score_2": 0.6314285714285715, "consistency_score_3": 0.49257142857142855, "consistency_score_4": 0.4123809523809523, "consistency_score_5": 0.3574603174603175, "consistency_score_6": 0.3152380952380952, "consistency_score_7": 0.28, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5533333333333333, "Filipino,Chinese": 0.54, "Filipino,Spanish": 0.6066666666666667, "Filipino,Malay": 0.58, "Filipino,Indonesian": 0.6, "Filipino,English": 0.5933333333333334, "Vietnamese,Chinese": 0.6466666666666666, "Vietnamese,Spanish": 0.66, "Vietnamese,Malay": 0.5866666666666667, "Vietnamese,Indonesian": 0.6466666666666666, "Vietnamese,English": 0.6866666666666666, "Chinese,Spanish": 0.6533333333333333, "Chinese,Malay": 0.6, "Chinese,Indonesian": 0.6666666666666666, "Chinese,English": 0.6333333333333333, "Spanish,Malay": 0.68, "Spanish,Indonesian": 0.6933333333333334, "Spanish,English": 0.7533333333333333, "Malay,Indonesian": 0.5933333333333334, "Malay,English": 0.6066666666666667, "Indonesian,English": 0.68 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.43333333333333335, "Filipino,Vietnamese,Spanish": 0.46, "Filipino,Vietnamese,Malay": 0.43333333333333335, "Filipino,Vietnamese,Indonesian": 0.46, "Filipino,Vietnamese,English": 0.4666666666666667, "Filipino,Chinese,Spanish": 0.4533333333333333, "Filipino,Chinese,Malay": 0.43333333333333335, "Filipino,Chinese,Indonesian": 0.4666666666666667, "Filipino,Chinese,English": 0.43333333333333335, "Filipino,Spanish,Malay": 0.4866666666666667, "Filipino,Spanish,Indonesian": 0.49333333333333335, "Filipino,Spanish,English": 0.5266666666666666, "Filipino,Malay,Indonesian": 0.44666666666666666, "Filipino,Malay,English": 0.44666666666666666, "Filipino,Indonesian,English": 0.4866666666666667, "Vietnamese,Chinese,Spanish": 0.5133333333333333, "Vietnamese,Chinese,Malay": 0.44666666666666666, "Vietnamese,Chinese,Indonesian": 0.5266666666666666, "Vietnamese,Chinese,English": 0.5266666666666666, "Vietnamese,Spanish,Malay": 0.49333333333333335, "Vietnamese,Spanish,Indonesian": 0.5266666666666666, "Vietnamese,Spanish,English": 0.5866666666666667, "Vietnamese,Malay,Indonesian": 0.46, "Vietnamese,Malay,English": 0.47333333333333333, "Vietnamese,Indonesian,English": 0.54, "Chinese,Spanish,Malay": 0.5, "Chinese,Spanish,Indonesian": 0.54, "Chinese,Spanish,English": 0.5533333333333333, "Chinese,Malay,Indonesian": 0.47333333333333333, "Chinese,Malay,English": 0.47333333333333333, "Chinese,Indonesian,English": 0.5333333333333333, "Spanish,Malay,Indonesian": 0.5133333333333333, "Spanish,Malay,English": 0.5666666666666667, "Spanish,Indonesian,English": 0.5866666666666667, "Malay,Indonesian,English": 0.48 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.38, "Filipino,Vietnamese,Chinese,Malay": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.3933333333333333, "Filipino,Vietnamese,Chinese,English": 0.38666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.38666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.4066666666666667, "Filipino,Vietnamese,Spanish,English": 0.43333333333333335, "Filipino,Vietnamese,Malay,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Malay,English": 0.36666666666666664, "Filipino,Vietnamese,Indonesian,English": 0.4066666666666667, "Filipino,Chinese,Spanish,Malay": 0.38666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.4066666666666667, "Filipino,Chinese,Spanish,English": 0.4066666666666667, "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, "Filipino,Chinese,Malay,English": 0.36, "Filipino,Chinese,Indonesian,English": 0.4, "Filipino,Spanish,Malay,Indonesian": 0.41333333333333333, "Filipino,Spanish,Malay,English": 0.44, "Filipino,Spanish,Indonesian,English": 0.4533333333333333, "Filipino,Malay,Indonesian,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.4533333333333333, "Vietnamese,Chinese,Spanish,English": 0.47333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Chinese,Malay,English": 0.3933333333333333, "Vietnamese,Chinese,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian": 0.42, "Vietnamese,Spanish,Malay,English": 0.4533333333333333, "Vietnamese,Spanish,Indonesian,English": 0.47333333333333333, "Vietnamese,Malay,Indonesian,English": 0.38666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, "Chinese,Spanish,Malay,English": 0.44, "Chinese,Spanish,Indonesian,English": 0.4866666666666667, "Chinese,Malay,Indonesian,English": 0.4066666666666667, "Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.36, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Malay,English": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.35333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.36, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.38, "Filipino,Vietnamese,Malay,Indonesian,English": 0.32, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.3466666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.35333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.38, "Filipino,Chinese,Malay,Indonesian,English": 0.32666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36666666666666664, "Vietnamese,Chinese,Spanish,Malay,English": 0.38, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.42, "Vietnamese,Chinese,Malay,Indonesian,English": 0.3466666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.38, "Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.28, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.32, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.34 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.28 } }, "AC3_2": 0.6197893350388912, "AC3_3": 0.5444614129537427, "AC3_4": 0.4916257995254071, "AC3_5": 0.4503788554061741, "AC3_6": 0.41533431512440777, "AC3_7": 0.38353697744879806 }, "prompt_2": { "overall_acc": 0.6057142857142858, "language_acc": { "Filipino": 0.49333333333333335, "Vietnamese": 0.6, "Chinese": 0.6066666666666667, "Spanish": 0.6266666666666667, "Malay": 0.5666666666666667, "Indonesian": 0.62, "English": 0.7266666666666667 }, "consistency_score_2": 0.5895238095238095, "consistency_score_3": 0.4453333333333332, "consistency_score_4": 0.36876190476190474, "consistency_score_5": 0.3177777777777778, "consistency_score_6": 0.28095238095238095, "consistency_score_7": 0.25333333333333335, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4533333333333333, "Filipino,Chinese": 0.47333333333333333, "Filipino,Spanish": 0.56, "Filipino,Malay": 0.49333333333333335, "Filipino,Indonesian": 0.5066666666666667, "Filipino,English": 0.58, "Vietnamese,Chinese": 0.6266666666666667, "Vietnamese,Spanish": 0.5666666666666667, "Vietnamese,Malay": 0.5333333333333333, "Vietnamese,Indonesian": 0.62, "Vietnamese,English": 0.6466666666666666, "Chinese,Spanish": 0.5933333333333334, "Chinese,Malay": 0.56, "Chinese,Indonesian": 0.6266666666666667, "Chinese,English": 0.5933333333333334, "Spanish,Malay": 0.68, "Spanish,Indonesian": 0.6333333333333333, "Spanish,English": 0.76, "Malay,Indonesian": 0.6066666666666667, "Malay,English": 0.6133333333333333, "Indonesian,English": 0.6533333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.36, "Filipino,Vietnamese,Spanish": 0.36, "Filipino,Vietnamese,Malay": 0.32666666666666666, "Filipino,Vietnamese,Indonesian": 0.36, "Filipino,Vietnamese,English": 0.38666666666666666, "Filipino,Chinese,Spanish": 0.3933333333333333, "Filipino,Chinese,Malay": 0.36666666666666664, "Filipino,Chinese,Indonesian": 0.38, "Filipino,Chinese,English": 0.3933333333333333, "Filipino,Spanish,Malay": 0.44, "Filipino,Spanish,Indonesian": 0.42, "Filipino,Spanish,English": 0.48, "Filipino,Malay,Indonesian": 0.4066666666666667, "Filipino,Malay,English": 0.4066666666666667, "Filipino,Indonesian,English": 0.41333333333333333, "Vietnamese,Chinese,Spanish": 0.4533333333333333, "Vietnamese,Chinese,Malay": 0.4266666666666667, "Vietnamese,Chinese,Indonesian": 0.49333333333333335, "Vietnamese,Chinese,English": 0.48, "Vietnamese,Spanish,Malay": 0.44, "Vietnamese,Spanish,Indonesian": 0.48, "Vietnamese,Spanish,English": 0.52, "Vietnamese,Malay,Indonesian": 0.44, "Vietnamese,Malay,English": 0.4533333333333333, "Vietnamese,Indonesian,English": 0.5066666666666667, "Chinese,Spanish,Malay": 0.4533333333333333, "Chinese,Spanish,Indonesian": 0.4866666666666667, "Chinese,Spanish,English": 0.5133333333333333, "Chinese,Malay,Indonesian": 0.48, "Chinese,Malay,English": 0.44, "Chinese,Indonesian,English": 0.47333333333333333, "Spanish,Malay,Indonesian": 0.5333333333333333, "Spanish,Malay,English": 0.56, "Spanish,Indonesian,English": 0.56, "Malay,Indonesian,English": 0.5 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Malay": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,Chinese,English": 0.32, "Filipino,Vietnamese,Spanish,Malay": 0.32, "Filipino,Vietnamese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Spanish,English": 0.3466666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Malay,English": 0.30666666666666664, "Filipino,Vietnamese,Indonesian,English": 0.3333333333333333, "Filipino,Chinese,Spanish,Malay": 0.34, "Filipino,Chinese,Spanish,Indonesian": 0.34, "Filipino,Chinese,Spanish,English": 0.36666666666666664, "Filipino,Chinese,Malay,Indonesian": 0.3333333333333333, "Filipino,Chinese,Malay,English": 0.31333333333333335, "Filipino,Chinese,Indonesian,English": 0.32666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.38666666666666666, "Filipino,Spanish,Malay,English": 0.4, "Filipino,Spanish,Indonesian,English": 0.38666666666666666, "Filipino,Malay,Indonesian,English": 0.35333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Spanish,English": 0.41333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Chinese,Malay,English": 0.38, "Vietnamese,Chinese,Indonesian,English": 0.42, "Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, "Vietnamese,Spanish,Malay,English": 0.41333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.44666666666666666, "Vietnamese,Malay,Indonesian,English": 0.4, "Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, "Chinese,Spanish,Malay,English": 0.4066666666666667, "Chinese,Spanish,Indonesian,English": 0.43333333333333335, "Chinese,Malay,Indonesian,English": 0.4, "Spanish,Malay,Indonesian,English": 0.47333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.28, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.2866666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Spanish,Malay,English": 0.30666666666666664, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, "Filipino,Chinese,Spanish,Malay,English": 0.31333333333333335, "Filipino,Chinese,Spanish,Indonesian,English": 0.31333333333333335, "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.35333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36, "Vietnamese,Chinese,Spanish,Malay,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.38, "Vietnamese,Chinese,Malay,Indonesian,English": 0.35333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.38, "Chinese,Spanish,Malay,Indonesian,English": 0.38 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.25333333333333335 } }, "AC3_2": 0.5975093909574081, "AC3_3": 0.5132874229305044, "AC3_4": 0.45842957663116474, "AC3_5": 0.41685802676819866, "AC3_6": 0.3838576031483604, "AC3_7": 0.3572505542821383 }, "prompt_3": { "overall_acc": 0.5714285714285714, "language_acc": { "Filipino": 0.5133333333333333, "Vietnamese": 0.5133333333333333, "Chinese": 0.5466666666666666, "Spanish": 0.62, "Malay": 0.5266666666666666, "Indonesian": 0.58, "English": 0.7 }, "consistency_score_2": 0.559047619047619, "consistency_score_3": 0.4116190476190476, "consistency_score_4": 0.33638095238095234, "consistency_score_5": 0.28888888888888886, "consistency_score_6": 0.2561904761904762, "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4666666666666667, "Filipino,Chinese": 0.47333333333333333, "Filipino,Spanish": 0.5466666666666666, "Filipino,Malay": 0.52, "Filipino,Indonesian": 0.56, "Filipino,English": 0.5733333333333334, "Vietnamese,Chinese": 0.58, "Vietnamese,Spanish": 0.5533333333333333, "Vietnamese,Malay": 0.58, "Vietnamese,Indonesian": 0.5266666666666666, "Vietnamese,English": 0.6, "Chinese,Spanish": 0.5866666666666667, "Chinese,Malay": 0.5266666666666666, "Chinese,Indonesian": 0.5666666666666667, "Chinese,English": 0.5733333333333334, "Spanish,Malay": 0.5733333333333334, "Spanish,Indonesian": 0.5466666666666666, "Spanish,English": 0.6733333333333333, "Malay,Indonesian": 0.5466666666666666, "Malay,English": 0.5333333333333333, "Indonesian,English": 0.6333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.35333333333333333, "Filipino,Vietnamese,Spanish": 0.36666666666666664, "Filipino,Vietnamese,Malay": 0.38, "Filipino,Vietnamese,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,English": 0.38, "Filipino,Chinese,Spanish": 0.38, "Filipino,Chinese,Malay": 0.3466666666666667, "Filipino,Chinese,Indonesian": 0.38, "Filipino,Chinese,English": 0.3933333333333333, "Filipino,Spanish,Malay": 0.4066666666666667, "Filipino,Spanish,Indonesian": 0.4, "Filipino,Spanish,English": 0.46, "Filipino,Malay,Indonesian": 0.4, "Filipino,Malay,English": 0.3933333333333333, "Filipino,Indonesian,English": 0.43333333333333335, "Vietnamese,Chinese,Spanish": 0.42, "Vietnamese,Chinese,Malay": 0.42, "Vietnamese,Chinese,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,English": 0.4533333333333333, "Vietnamese,Spanish,Malay": 0.43333333333333335, "Vietnamese,Spanish,Indonesian": 0.3933333333333333, "Vietnamese,Spanish,English": 0.4533333333333333, "Vietnamese,Malay,Indonesian": 0.3933333333333333, "Vietnamese,Malay,English": 0.43333333333333335, "Vietnamese,Indonesian,English": 0.44666666666666666, "Chinese,Spanish,Malay": 0.4066666666666667, "Chinese,Spanish,Indonesian": 0.4066666666666667, "Chinese,Spanish,English": 0.47333333333333333, "Chinese,Malay,Indonesian": 0.37333333333333335, "Chinese,Malay,English": 0.4, "Chinese,Indonesian,English": 0.4533333333333333, "Spanish,Malay,Indonesian": 0.41333333333333333, "Spanish,Malay,English": 0.46, "Spanish,Indonesian,English": 0.4866666666666667, "Malay,Indonesian,English": 0.43333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.3, "Filipino,Vietnamese,Chinese,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Chinese,English": 0.31333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.3333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Spanish,English": 0.32, "Filipino,Vietnamese,Malay,Indonesian": 0.3, "Filipino,Vietnamese,Malay,English": 0.31333333333333335, "Filipino,Vietnamese,Indonesian,English": 0.32, "Filipino,Chinese,Spanish,Malay": 0.30666666666666664, "Filipino,Chinese,Spanish,Indonesian": 0.31333333333333335, "Filipino,Chinese,Spanish,English": 0.3466666666666667, "Filipino,Chinese,Malay,Indonesian": 0.2866666666666667, "Filipino,Chinese,Malay,English": 0.3, "Filipino,Chinese,Indonesian,English": 0.3333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.34, "Filipino,Spanish,Malay,English": 0.36666666666666664, "Filipino,Spanish,Indonesian,English": 0.37333333333333335, "Filipino,Malay,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish,Malay": 0.3466666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.34, "Vietnamese,Chinese,Spanish,English": 0.38, "Vietnamese,Chinese,Malay,Indonesian": 0.32, "Vietnamese,Chinese,Malay,English": 0.36, "Vietnamese,Chinese,Indonesian,English": 0.38, "Vietnamese,Spanish,Malay,Indonesian": 0.34, "Vietnamese,Spanish,Malay,English": 0.38666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.37333333333333335, "Vietnamese,Malay,Indonesian,English": 0.34, "Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Chinese,Spanish,Malay,English": 0.36, "Chinese,Spanish,Indonesian,English": 0.38, "Chinese,Malay,Indonesian,English": 0.3333333333333333, "Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,Malay,English": 0.3, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2866666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.3, "Filipino,Chinese,Malay,Indonesian,English": 0.26, "Filipino,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, "Vietnamese,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, "AC3_2": 0.5651702972180286, "AC3_3": 0.47853406034936163, "AC3_4": 0.4234758107560615, "AC3_5": 0.3837638375937691, "AC3_6": 0.35377280942626615, "AC3_7": 0.3313609467043871 }, "prompt_4": { "overall_acc": 0.6038095238095238, "language_acc": { "Filipino": 0.54, "Vietnamese": 0.56, "Chinese": 0.5866666666666667, "Spanish": 0.62, "Malay": 0.5866666666666667, "Indonesian": 0.6066666666666667, "English": 0.7266666666666667 }, "consistency_score_2": 0.6155555555555553, "consistency_score_3": 0.47142857142857153, "consistency_score_4": 0.3900952380952382, "consistency_score_5": 0.33492063492063495, "consistency_score_6": 0.29333333333333333, "consistency_score_7": 0.26, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5333333333333333, "Filipino,Chinese": 0.5333333333333333, "Filipino,Spanish": 0.5666666666666667, "Filipino,Malay": 0.49333333333333335, "Filipino,Indonesian": 0.5533333333333333, "Filipino,English": 0.5933333333333334, "Vietnamese,Chinese": 0.6266666666666667, "Vietnamese,Spanish": 0.66, "Vietnamese,Malay": 0.56, "Vietnamese,Indonesian": 0.64, "Vietnamese,English": 0.6466666666666666, "Chinese,Spanish": 0.64, "Chinese,Malay": 0.6066666666666667, "Chinese,Indonesian": 0.7, "Chinese,English": 0.6466666666666666, "Spanish,Malay": 0.6133333333333333, "Spanish,Indonesian": 0.6733333333333333, "Spanish,English": 0.7, "Malay,Indonesian": 0.6533333333333333, "Malay,English": 0.5866666666666667, "Indonesian,English": 0.7 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.4066666666666667, "Filipino,Vietnamese,Spanish": 0.4266666666666667, "Filipino,Vietnamese,Malay": 0.36666666666666664, "Filipino,Vietnamese,Indonesian": 0.42, "Filipino,Vietnamese,English": 0.4266666666666667, "Filipino,Chinese,Spanish": 0.41333333333333333, "Filipino,Chinese,Malay": 0.38666666666666666, "Filipino,Chinese,Indonesian": 0.4533333333333333, "Filipino,Chinese,English": 0.44, "Filipino,Spanish,Malay": 0.3933333333333333, "Filipino,Spanish,Indonesian": 0.44, "Filipino,Spanish,English": 0.4666666666666667, "Filipino,Malay,Indonesian": 0.4266666666666667, "Filipino,Malay,English": 0.3933333333333333, "Filipino,Indonesian,English": 0.46, "Vietnamese,Chinese,Spanish": 0.5, "Vietnamese,Chinese,Malay": 0.44, "Vietnamese,Chinese,Indonesian": 0.5333333333333333, "Vietnamese,Chinese,English": 0.5066666666666667, "Vietnamese,Spanish,Malay": 0.46, "Vietnamese,Spanish,Indonesian": 0.5333333333333333, "Vietnamese,Spanish,English": 0.5333333333333333, "Vietnamese,Malay,Indonesian": 0.47333333333333333, "Vietnamese,Malay,English": 0.44666666666666666, "Vietnamese,Indonesian,English": 0.5266666666666666, "Chinese,Spanish,Malay": 0.4866666666666667, "Chinese,Spanish,Indonesian": 0.5533333333333333, "Chinese,Spanish,English": 0.5466666666666666, "Chinese,Malay,Indonesian": 0.5266666666666666, "Chinese,Malay,English": 0.4533333333333333, "Chinese,Indonesian,English": 0.54, "Spanish,Malay,Indonesian": 0.52, "Spanish,Malay,English": 0.5133333333333333, "Spanish,Indonesian,English": 0.5733333333333334, "Malay,Indonesian,English": 0.5133333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.34, "Filipino,Vietnamese,Chinese,Malay": 0.32, "Filipino,Vietnamese,Chinese,Indonesian": 0.37333333333333335, "Filipino,Vietnamese,Chinese,English": 0.36, "Filipino,Vietnamese,Spanish,Malay": 0.32, "Filipino,Vietnamese,Spanish,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Spanish,English": 0.36666666666666664, "Filipino,Vietnamese,Malay,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,Malay,English": 0.32, "Filipino,Vietnamese,Indonesian,English": 0.37333333333333335, "Filipino,Chinese,Spanish,Malay": 0.3333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.38, "Filipino,Chinese,Spanish,English": 0.36666666666666664, "Filipino,Chinese,Malay,Indonesian": 0.37333333333333335, "Filipino,Chinese,Malay,English": 0.32, "Filipino,Chinese,Indonesian,English": 0.38666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Spanish,Malay,English": 0.3466666666666667, "Filipino,Spanish,Indonesian,English": 0.3933333333333333, "Filipino,Malay,Indonesian,English": 0.36, "Vietnamese,Chinese,Spanish,Malay": 0.3933333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,Spanish,English": 0.44666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Malay,English": 0.38, "Vietnamese,Chinese,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian": 0.4266666666666667, "Vietnamese,Spanish,Malay,English": 0.42, "Vietnamese,Spanish,Indonesian,English": 0.48, "Vietnamese,Malay,Indonesian,English": 0.41333333333333333, "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Chinese,Spanish,Malay,English": 0.41333333333333333, "Chinese,Spanish,Indonesian,English": 0.4866666666666667, "Chinese,Malay,Indonesian,English": 0.43333333333333335, "Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2866666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.31333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Malay,English": 0.28, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,Spanish,Malay,English": 0.29333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.34, "Filipino,Vietnamese,Malay,Indonesian,English": 0.3, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.34, "Filipino,Chinese,Malay,Indonesian,English": 0.31333333333333335, "Filipino,Spanish,Malay,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38, "Vietnamese,Chinese,Spanish,Malay,English": 0.36, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4266666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.37333333333333335, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4, "Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.26, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26 } }, "AC3_2": 0.609625965410388, "AC3_3": 0.5294698215374972, "AC3_4": 0.4739754329771891, "AC3_5": 0.43085495268933915, "AC3_6": 0.3948478414280336, "AC3_7": 0.3634840131883508 }, "prompt_5": { "overall_acc": 0.5923809523809523, "language_acc": { "Filipino": 0.56, "Vietnamese": 0.5866666666666667, "Chinese": 0.6066666666666667, "Spanish": 0.56, "Malay": 0.5266666666666666, "Indonesian": 0.6066666666666667, "English": 0.7 }, "consistency_score_2": 0.5876190476190475, "consistency_score_3": 0.4432380952380953, "consistency_score_4": 0.3651428571428571, "consistency_score_5": 0.31238095238095237, "consistency_score_6": 0.27238095238095245, "consistency_score_7": 0.24, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.56, "Filipino,Chinese": 0.5133333333333333, "Filipino,Spanish": 0.54, "Filipino,Malay": 0.49333333333333335, "Filipino,Indonesian": 0.54, "Filipino,English": 0.6066666666666667, "Vietnamese,Chinese": 0.6333333333333333, "Vietnamese,Spanish": 0.6466666666666666, "Vietnamese,Malay": 0.5466666666666666, "Vietnamese,Indonesian": 0.6133333333333333, "Vietnamese,English": 0.6333333333333333, "Chinese,Spanish": 0.5933333333333334, "Chinese,Malay": 0.5266666666666666, "Chinese,Indonesian": 0.6133333333333333, "Chinese,English": 0.6133333333333333, "Spanish,Malay": 0.6, "Spanish,Indonesian": 0.62, "Spanish,English": 0.66, "Malay,Indonesian": 0.54, "Malay,English": 0.5533333333333333, "Indonesian,English": 0.6933333333333334 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.41333333333333333, "Filipino,Vietnamese,Spanish": 0.4266666666666667, "Filipino,Vietnamese,Malay": 0.38, "Filipino,Vietnamese,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,English": 0.44666666666666666, "Filipino,Chinese,Spanish": 0.41333333333333333, "Filipino,Chinese,Malay": 0.36, "Filipino,Chinese,Indonesian": 0.41333333333333333, "Filipino,Chinese,English": 0.42, "Filipino,Spanish,Malay": 0.3933333333333333, "Filipino,Spanish,Indonesian": 0.44, "Filipino,Spanish,English": 0.4666666666666667, "Filipino,Malay,Indonesian": 0.38, "Filipino,Malay,English": 0.38, "Filipino,Indonesian,English": 0.46, "Vietnamese,Chinese,Spanish": 0.4866666666666667, "Vietnamese,Chinese,Malay": 0.42, "Vietnamese,Chinese,Indonesian": 0.48, "Vietnamese,Chinese,English": 0.49333333333333335, "Vietnamese,Spanish,Malay": 0.44666666666666666, "Vietnamese,Spanish,Indonesian": 0.47333333333333333, "Vietnamese,Spanish,English": 0.52, "Vietnamese,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Malay,English": 0.4266666666666667, "Vietnamese,Indonesian,English": 0.5, "Chinese,Spanish,Malay": 0.43333333333333335, "Chinese,Spanish,Indonesian": 0.47333333333333333, "Chinese,Spanish,English": 0.49333333333333335, "Chinese,Malay,Indonesian": 0.41333333333333333, "Chinese,Malay,English": 0.4266666666666667, "Chinese,Indonesian,English": 0.5, "Spanish,Malay,Indonesian": 0.44, "Spanish,Malay,English": 0.48, "Spanish,Indonesian,English": 0.5266666666666666, "Malay,Indonesian,English": 0.44666666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.3, "Filipino,Vietnamese,Chinese,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Chinese,English": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Malay": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.36, "Filipino,Vietnamese,Spanish,English": 0.38666666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.32, "Filipino,Vietnamese,Malay,English": 0.32, "Filipino,Vietnamese,Indonesian,English": 0.38, "Filipino,Chinese,Spanish,Malay": 0.32, "Filipino,Chinese,Spanish,Indonesian": 0.36666666666666664, "Filipino,Chinese,Spanish,English": 0.36666666666666664, "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, "Filipino,Chinese,Malay,English": 0.30666666666666664, "Filipino,Chinese,Indonesian,English": 0.37333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.3466666666666667, "Filipino,Spanish,Malay,English": 0.35333333333333333, "Filipino,Spanish,Indonesian,English": 0.3933333333333333, "Filipino,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.36666666666666664, "Vietnamese,Chinese,Spanish,Indonesian": 0.4, "Vietnamese,Chinese,Spanish,English": 0.4266666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.36, "Vietnamese,Chinese,Malay,English": 0.36, "Vietnamese,Chinese,Indonesian,English": 0.4266666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.36666666666666664, "Vietnamese,Spanish,Malay,English": 0.4, "Vietnamese,Spanish,Indonesian,English": 0.4266666666666667, "Vietnamese,Malay,Indonesian,English": 0.36666666666666664, "Chinese,Spanish,Malay,Indonesian": 0.36666666666666664, "Chinese,Spanish,Malay,English": 0.38, "Chinese,Spanish,Indonesian,English": 0.42, "Chinese,Malay,Indonesian,English": 0.36666666666666664, "Spanish,Malay,Indonesian,English": 0.4 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2733333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.32, "Filipino,Vietnamese,Chinese,Spanish,English": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.30666666666666664, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.3333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.28, "Filipino,Spanish,Malay,Indonesian,English": 0.31333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.36666666666666664, "Vietnamese,Chinese,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.34, "Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.3, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2733333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.24 } }, "AC3_2": 0.5899903915868815, "AC3_3": 0.5070702505268974, "AC3_4": 0.4517980050112149, "AC3_5": 0.4090546365462674, "AC3_6": 0.37317390387964927, "AC3_7": 0.34160183062257643 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.4926948051948052, "language_acc": { "Indonesian": 0.4772727272727273, "English": 0.5397727272727273, "Filipino": 0.4602272727272727, "Spanish": 0.48295454545454547, "Chinese": 0.48863636363636365, "Malay": 0.4943181818181818, "Vietnamese": 0.5056818181818182 }, "consistency_score_2": 0.6130952380952381, "consistency_score_3": 0.45925324675324664, "consistency_score_4": 0.37370129870129876, "consistency_score_5": 0.31764069264069256, "consistency_score_6": 0.2767857142857143, "consistency_score_7": 0.24431818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6647727272727273, "Indonesian,Filipino": 0.6477272727272727, "Indonesian,Spanish": 0.6420454545454546, "Indonesian,Chinese": 0.5568181818181818, "Indonesian,Malay": 0.7272727272727273, "Indonesian,Vietnamese": 0.6761363636363636, "English,Filipino": 0.6136363636363636, "English,Spanish": 0.7272727272727273, "English,Chinese": 0.5681818181818182, "English,Malay": 0.6875, "English,Vietnamese": 0.6420454545454546, "Filipino,Spanish": 0.5965909090909091, "Filipino,Chinese": 0.5397727272727273, "Filipino,Malay": 0.5511363636363636, "Filipino,Vietnamese": 0.5681818181818182, "Spanish,Chinese": 0.4943181818181818, "Spanish,Malay": 0.625, "Spanish,Vietnamese": 0.6704545454545454, "Chinese,Malay": 0.5340909090909091, "Chinese,Vietnamese": 0.5284090909090909, "Malay,Vietnamese": 0.6136363636363636 }, "3_combine": { "Indonesian,English,Filipino": 0.5, "Indonesian,English,Spanish": 0.5454545454545454, "Indonesian,English,Chinese": 0.4431818181818182, "Indonesian,English,Malay": 0.5681818181818182, "Indonesian,English,Vietnamese": 0.5284090909090909, "Indonesian,Filipino,Spanish": 0.4715909090909091, "Indonesian,Filipino,Chinese": 0.4318181818181818, "Indonesian,Filipino,Malay": 0.5, "Indonesian,Filipino,Vietnamese": 0.4659090909090909, "Indonesian,Spanish,Chinese": 0.3977272727272727, "Indonesian,Spanish,Malay": 0.5340909090909091, "Indonesian,Spanish,Vietnamese": 0.5227272727272727, "Indonesian,Chinese,Malay": 0.44886363636363635, "Indonesian,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,Malay,Vietnamese": 0.5454545454545454, "English,Filipino,Spanish": 0.48863636363636365, "English,Filipino,Chinese": 0.4147727272727273, "English,Filipino,Malay": 0.4659090909090909, "English,Filipino,Vietnamese": 0.4431818181818182, "English,Spanish,Chinese": 0.4318181818181818, "English,Spanish,Malay": 0.5511363636363636, "English,Spanish,Vietnamese": 0.5511363636363636, "English,Chinese,Malay": 0.4375, "English,Chinese,Vietnamese": 0.39204545454545453, "English,Malay,Vietnamese": 0.5056818181818182, "Filipino,Spanish,Chinese": 0.3693181818181818, "Filipino,Spanish,Malay": 0.4318181818181818, "Filipino,Spanish,Vietnamese": 0.45454545454545453, "Filipino,Chinese,Malay": 0.375, "Filipino,Chinese,Vietnamese": 0.375, "Filipino,Malay,Vietnamese": 0.4090909090909091, "Spanish,Chinese,Malay": 0.38636363636363635, "Spanish,Chinese,Vietnamese": 0.39204545454545453, "Spanish,Malay,Vietnamese": 0.48863636363636365, "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.42045454545454547, "Indonesian,English,Filipino,Chinese": 0.3693181818181818, "Indonesian,English,Filipino,Malay": 0.42613636363636365, "Indonesian,English,Filipino,Vietnamese": 0.39204545454545453, "Indonesian,English,Spanish,Chinese": 0.36363636363636365, "Indonesian,English,Spanish,Malay": 0.48863636363636365, "Indonesian,English,Spanish,Vietnamese": 0.4602272727272727, "Indonesian,English,Chinese,Malay": 0.38636363636363635, "Indonesian,English,Chinese,Vietnamese": 0.3465909090909091, "Indonesian,English,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.32954545454545453, "Indonesian,Filipino,Spanish,Malay": 0.4034090909090909, "Indonesian,Filipino,Spanish,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Chinese,Malay": 0.35795454545454547, "Indonesian,Filipino,Chinese,Vietnamese": 0.32954545454545453, "Indonesian,Filipino,Malay,Vietnamese": 0.38636363636363635, "Indonesian,Spanish,Chinese,Malay": 0.3522727272727273, "Indonesian,Spanish,Chinese,Vietnamese": 0.3409090909090909, "Indonesian,Spanish,Malay,Vietnamese": 0.45454545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.35795454545454547, "English,Filipino,Spanish,Chinese": 0.3352272727272727, "English,Filipino,Spanish,Malay": 0.4034090909090909, "English,Filipino,Spanish,Vietnamese": 0.39204545454545453, "English,Filipino,Chinese,Malay": 0.3409090909090909, "English,Filipino,Chinese,Vietnamese": 0.29545454545454547, "English,Filipino,Malay,Vietnamese": 0.3806818181818182, "English,Spanish,Chinese,Malay": 0.36363636363636365, "English,Spanish,Chinese,Vietnamese": 0.3522727272727273, "English,Spanish,Malay,Vietnamese": 0.44886363636363635, "English,Chinese,Malay,Vietnamese": 0.32954545454545453, "Filipino,Spanish,Chinese,Malay": 0.32386363636363635, "Filipino,Spanish,Chinese,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.36363636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.30113636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.3068181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.375, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.3522727272727273, "Indonesian,English,Filipino,Chinese,Malay": 0.32386363636363635, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.35795454545454547, "Indonesian,English,Spanish,Chinese,Malay": 0.32954545454545453, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3125, "Indonesian,English,Spanish,Malay,Vietnamese": 0.42613636363636365, "Indonesian,English,Chinese,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3068181818181818, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.3465909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2897727272727273, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818, "English,Filipino,Spanish,Chinese,Malay": 0.30113636363636365, "English,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.3465909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.2727272727272727, "English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.2840909090909091, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.32954545454545453, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182 } }, "AC3_2": 0.5463403124396696, "AC3_3": 0.47538663156537675, "AC3_4": 0.4250265847583037, "AC3_5": 0.38625956676281836, "AC3_6": 0.3544492163493555, "AC3_7": 0.32665448533813757 }, "prompt_2": { "overall_acc": 0.4675324675324675, "language_acc": { "Indonesian": 0.4772727272727273, "English": 0.5454545454545454, "Filipino": 0.42613636363636365, "Spanish": 0.48295454545454547, "Chinese": 0.42613636363636365, "Malay": 0.44886363636363635, "Vietnamese": 0.4659090909090909 }, "consistency_score_2": 0.5768398268398267, "consistency_score_3": 0.4141233766233765, "consistency_score_4": 0.32678571428571423, "consistency_score_5": 0.2719155844155844, "consistency_score_6": 0.23538961038961034, "consistency_score_7": 0.21022727272727273, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6534090909090909, "Indonesian,Filipino": 0.5568181818181818, "Indonesian,Spanish": 0.6306818181818182, "Indonesian,Chinese": 0.5284090909090909, "Indonesian,Malay": 0.7045454545454546, "Indonesian,Vietnamese": 0.6079545454545454, "English,Filipino": 0.5227272727272727, "English,Spanish": 0.7443181818181818, "English,Chinese": 0.5795454545454546, "English,Malay": 0.6477272727272727, "English,Vietnamese": 0.5795454545454546, "Filipino,Spanish": 0.5170454545454546, "Filipino,Chinese": 0.5056818181818182, "Filipino,Malay": 0.5511363636363636, "Filipino,Vietnamese": 0.48863636363636365, "Spanish,Chinese": 0.5397727272727273, "Spanish,Malay": 0.5965909090909091, "Spanish,Vietnamese": 0.6022727272727273, "Chinese,Malay": 0.5056818181818182, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.5625 }, "3_combine": { "Indonesian,English,Filipino": 0.42045454545454547, "Indonesian,English,Spanish": 0.5340909090909091, "Indonesian,English,Chinese": 0.4431818181818182, "Indonesian,English,Malay": 0.5284090909090909, "Indonesian,English,Vietnamese": 0.4602272727272727, "Indonesian,Filipino,Spanish": 0.3977272727272727, "Indonesian,Filipino,Chinese": 0.36363636363636365, "Indonesian,Filipino,Malay": 0.44886363636363635, "Indonesian,Filipino,Vietnamese": 0.38636363636363635, "Indonesian,Spanish,Chinese": 0.38636363636363635, "Indonesian,Spanish,Malay": 0.5, "Indonesian,Spanish,Vietnamese": 0.45454545454545453, "Indonesian,Chinese,Malay": 0.4090909090909091, "Indonesian,Chinese,Vietnamese": 0.36363636363636365, "Indonesian,Malay,Vietnamese": 0.4659090909090909, "English,Filipino,Spanish": 0.42613636363636365, "English,Filipino,Chinese": 0.375, "English,Filipino,Malay": 0.4090909090909091, "English,Filipino,Vietnamese": 0.3409090909090909, "English,Spanish,Chinese": 0.4715909090909091, "English,Spanish,Malay": 0.5397727272727273, "English,Spanish,Vietnamese": 0.5, "English,Chinese,Malay": 0.4090909090909091, "English,Chinese,Vietnamese": 0.375, "English,Malay,Vietnamese": 0.42613636363636365, "Filipino,Spanish,Chinese": 0.3522727272727273, "Filipino,Spanish,Malay": 0.39204545454545453, "Filipino,Spanish,Vietnamese": 0.35795454545454547, "Filipino,Chinese,Malay": 0.35795454545454547, "Filipino,Chinese,Vietnamese": 0.3181818181818182, "Filipino,Malay,Vietnamese": 0.35795454545454547, "Spanish,Chinese,Malay": 0.3806818181818182, "Spanish,Chinese,Vietnamese": 0.3693181818181818, "Spanish,Malay,Vietnamese": 0.4318181818181818, "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.3522727272727273, "Indonesian,English,Filipino,Chinese": 0.3181818181818182, "Indonesian,English,Filipino,Malay": 0.35795454545454547, "Indonesian,English,Filipino,Vietnamese": 0.3125, "Indonesian,English,Spanish,Chinese": 0.3693181818181818, "Indonesian,English,Spanish,Malay": 0.4602272727272727, "Indonesian,English,Spanish,Vietnamese": 0.3977272727272727, "Indonesian,English,Chinese,Malay": 0.35795454545454547, "Indonesian,English,Chinese,Vietnamese": 0.32386363636363635, "Indonesian,English,Malay,Vietnamese": 0.3806818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.29545454545454547, "Indonesian,Filipino,Spanish,Malay": 0.3522727272727273, "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, "Indonesian,Filipino,Chinese,Malay": 0.3125, "Indonesian,Filipino,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.32386363636363635, "Indonesian,Spanish,Chinese,Malay": 0.3409090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.30113636363636365, "English,Filipino,Spanish,Chinese": 0.32954545454545453, "English,Filipino,Spanish,Malay": 0.35795454545454547, "English,Filipino,Spanish,Vietnamese": 0.3068181818181818, "English,Filipino,Chinese,Malay": 0.3068181818181818, "English,Filipino,Chinese,Vietnamese": 0.26136363636363635, "English,Filipino,Malay,Vietnamese": 0.2840909090909091, "English,Spanish,Chinese,Malay": 0.36363636363636365, "English,Spanish,Chinese,Vietnamese": 0.3352272727272727, "English,Spanish,Malay,Vietnamese": 0.3977272727272727, "English,Chinese,Malay,Vietnamese": 0.2840909090909091, "Filipino,Spanish,Chinese,Malay": 0.29545454545454547, "Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, "Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2840909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.3181818181818182, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.2784090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.25, "Indonesian,English,Filipino,Malay,Vietnamese": 0.26704545454545453, "Indonesian,English,Spanish,Chinese,Malay": 0.32386363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3522727272727273, "Indonesian,English,Chinese,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2727272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese,Malay": 0.2840909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.25, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273 } }, "AC3_2": 0.5164659174527346, "AC3_3": 0.4392102097367143, "AC3_4": 0.3846894980684131, "AC3_5": 0.343849344900539, "AC3_6": 0.3131279805199235, "AC3_7": 0.29003810556417425 }, "prompt_3": { "overall_acc": 0.4521103896103896, "language_acc": { "Indonesian": 0.4375, "English": 0.5340909090909091, "Filipino": 0.38636363636363635, "Spanish": 0.4602272727272727, "Chinese": 0.4431818181818182, "Malay": 0.4659090909090909, "Vietnamese": 0.4375 }, "consistency_score_2": 0.5446428571428573, "consistency_score_3": 0.376461038961039, "consistency_score_4": 0.29188311688311686, "consistency_score_5": 0.24296536796536794, "consistency_score_6": 0.2126623376623376, "consistency_score_7": 0.19318181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5852272727272727, "Indonesian,Filipino": 0.5340909090909091, "Indonesian,Spanish": 0.5568181818181818, "Indonesian,Chinese": 0.5170454545454546, "Indonesian,Malay": 0.6534090909090909, "Indonesian,Vietnamese": 0.5170454545454546, "English,Filipino": 0.5113636363636364, "English,Spanish": 0.6818181818181818, "English,Chinese": 0.5681818181818182, "English,Malay": 0.6136363636363636, "English,Vietnamese": 0.5284090909090909, "Filipino,Spanish": 0.4715909090909091, "Filipino,Chinese": 0.4772727272727273, "Filipino,Malay": 0.5340909090909091, "Filipino,Vietnamese": 0.48863636363636365, "Spanish,Chinese": 0.5170454545454546, "Spanish,Malay": 0.5909090909090909, "Spanish,Vietnamese": 0.5909090909090909, "Chinese,Malay": 0.4659090909090909, "Chinese,Vietnamese": 0.5056818181818182, "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.375, "Indonesian,English,Spanish": 0.45454545454545453, "Indonesian,English,Chinese": 0.4034090909090909, "Indonesian,English,Malay": 0.4715909090909091, "Indonesian,English,Vietnamese": 0.3806818181818182, "Indonesian,Filipino,Spanish": 0.3465909090909091, "Indonesian,Filipino,Chinese": 0.3352272727272727, "Indonesian,Filipino,Malay": 0.42613636363636365, "Indonesian,Filipino,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Chinese": 0.3522727272727273, "Indonesian,Spanish,Malay": 0.4431818181818182, "Indonesian,Spanish,Vietnamese": 0.38636363636363635, "Indonesian,Chinese,Malay": 0.36363636363636365, "Indonesian,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,Malay,Vietnamese": 0.3977272727272727, "English,Filipino,Spanish": 0.38636363636363635, "English,Filipino,Chinese": 0.3465909090909091, "English,Filipino,Malay": 0.3806818181818182, "English,Filipino,Vietnamese": 0.3352272727272727, "English,Spanish,Chinese": 0.42613636363636365, "English,Spanish,Malay": 0.48863636363636365, "English,Spanish,Vietnamese": 0.4431818181818182, "English,Chinese,Malay": 0.3806818181818182, "English,Chinese,Vietnamese": 0.375, "English,Malay,Vietnamese": 0.38636363636363635, "Filipino,Spanish,Chinese": 0.30113636363636365, "Filipino,Spanish,Malay": 0.36363636363636365, "Filipino,Spanish,Vietnamese": 0.3409090909090909, "Filipino,Chinese,Malay": 0.32386363636363635, "Filipino,Chinese,Vietnamese": 0.30113636363636365, "Filipino,Malay,Vietnamese": 0.3409090909090909, "Spanish,Chinese,Malay": 0.3522727272727273, "Spanish,Chinese,Vietnamese": 0.3693181818181818, "Spanish,Malay,Vietnamese": 0.4090909090909091, "Chinese,Malay,Vietnamese": 0.3181818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.30113636363636365, "Indonesian,English,Filipino,Chinese": 0.2840909090909091, "Indonesian,English,Filipino,Malay": 0.32954545454545453, "Indonesian,English,Filipino,Vietnamese": 0.2727272727272727, "Indonesian,English,Spanish,Chinese": 0.3181818181818182, "Indonesian,English,Spanish,Malay": 0.38636363636363635, "Indonesian,English,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,English,Chinese,Malay": 0.32954545454545453, "Indonesian,English,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,English,Malay,Vietnamese": 0.32954545454545453, "Indonesian,Filipino,Spanish,Chinese": 0.25, "Indonesian,Filipino,Spanish,Malay": 0.3125, "Indonesian,Filipino,Spanish,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Chinese,Malay": 0.2840909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Chinese,Malay": 0.29545454545454547, "Indonesian,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese": 0.2784090909090909, "English,Filipino,Spanish,Malay": 0.3181818181818182, "English,Filipino,Spanish,Vietnamese": 0.2897727272727273, "English,Filipino,Chinese,Malay": 0.2727272727272727, "English,Filipino,Chinese,Vietnamese": 0.24431818181818182, "English,Filipino,Malay,Vietnamese": 0.2840909090909091, "English,Spanish,Chinese,Malay": 0.32954545454545453, "English,Spanish,Chinese,Vietnamese": 0.32386363636363635, "English,Spanish,Malay,Vietnamese": 0.3465909090909091, "English,Chinese,Malay,Vietnamese": 0.2727272727272727, "Filipino,Spanish,Chinese,Malay": 0.2556818181818182, "Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.23295454545454544, "Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.23295454545454544, "Indonesian,English,Filipino,Spanish,Malay": 0.2784090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.2556818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,English,Filipino,Malay,Vietnamese": 0.26136363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.2840909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.29545454545454547, "Indonesian,English,Chinese,Malay,Vietnamese": 0.25, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23863636363636365, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "English,Filipino,Spanish,Chinese,Malay": 0.24431818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.25, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.22727272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818 } }, "AC3_2": 0.49408154950836486, "AC3_3": 0.4108322858071933, "AC3_4": 0.35474339097107555, "AC3_5": 0.31607250277284515, "AC3_6": 0.28926232493309767, "AC3_7": 0.27069754141030883 }, "prompt_4": { "overall_acc": 0.4748376623376624, "language_acc": { "Indonesian": 0.4659090909090909, "English": 0.5511363636363636, "Filipino": 0.42613636363636365, "Spanish": 0.4431818181818182, "Chinese": 0.4602272727272727, "Malay": 0.5056818181818182, "Vietnamese": 0.4715909090909091 }, "consistency_score_2": 0.5817099567099566, "consistency_score_3": 0.4222402597402597, "consistency_score_4": 0.335064935064935, "consistency_score_5": 0.27949134199134196, "consistency_score_6": 0.24188311688311687, "consistency_score_7": 0.2159090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6193181818181818, "Indonesian,Filipino": 0.5852272727272727, "Indonesian,Spanish": 0.5795454545454546, "Indonesian,Chinese": 0.5056818181818182, "Indonesian,Malay": 0.6931818181818182, "Indonesian,Vietnamese": 0.5965909090909091, "English,Filipino": 0.5170454545454546, "English,Spanish": 0.7045454545454546, "English,Chinese": 0.5625, "English,Malay": 0.6590909090909091, "English,Vietnamese": 0.6363636363636364, "Filipino,Spanish": 0.5454545454545454, "Filipino,Chinese": 0.5454545454545454, "Filipino,Malay": 0.5568181818181818, "Filipino,Vietnamese": 0.5397727272727273, "Spanish,Chinese": 0.4943181818181818, "Spanish,Malay": 0.5965909090909091, "Spanish,Vietnamese": 0.6704545454545454, "Chinese,Malay": 0.4943181818181818, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.5965909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.4147727272727273, "Indonesian,English,Spanish": 0.48863636363636365, "Indonesian,English,Chinese": 0.39204545454545453, "Indonesian,English,Malay": 0.5227272727272727, "Indonesian,English,Vietnamese": 0.4715909090909091, "Indonesian,Filipino,Spanish": 0.3977272727272727, "Indonesian,Filipino,Chinese": 0.3806818181818182, "Indonesian,Filipino,Malay": 0.4602272727272727, "Indonesian,Filipino,Vietnamese": 0.3977272727272727, "Indonesian,Spanish,Chinese": 0.3409090909090909, "Indonesian,Spanish,Malay": 0.4772727272727273, "Indonesian,Spanish,Vietnamese": 0.4659090909090909, "Indonesian,Chinese,Malay": 0.4034090909090909, "Indonesian,Chinese,Vietnamese": 0.35795454545454547, "Indonesian,Malay,Vietnamese": 0.48863636363636365, "English,Filipino,Spanish": 0.42613636363636365, "English,Filipino,Chinese": 0.3806818181818182, "English,Filipino,Malay": 0.42045454545454547, "English,Filipino,Vietnamese": 0.4034090909090909, "English,Spanish,Chinese": 0.42613636363636365, "English,Spanish,Malay": 0.5113636363636364, "English,Spanish,Vietnamese": 0.5454545454545454, "English,Chinese,Malay": 0.42045454545454547, "English,Chinese,Vietnamese": 0.3977272727272727, "English,Malay,Vietnamese": 0.48295454545454547, "Filipino,Spanish,Chinese": 0.36363636363636365, "Filipino,Spanish,Malay": 0.4034090909090909, "Filipino,Spanish,Vietnamese": 0.42613636363636365, "Filipino,Chinese,Malay": 0.35795454545454547, "Filipino,Chinese,Vietnamese": 0.36363636363636365, "Filipino,Malay,Vietnamese": 0.3977272727272727, "Spanish,Chinese,Malay": 0.36363636363636365, "Spanish,Chinese,Vietnamese": 0.39204545454545453, "Spanish,Malay,Vietnamese": 0.4715909090909091, "Chinese,Malay,Vietnamese": 0.36363636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.3409090909090909, "Indonesian,English,Filipino,Chinese": 0.30113636363636365, "Indonesian,English,Filipino,Malay": 0.36363636363636365, "Indonesian,English,Filipino,Vietnamese": 0.32954545454545453, "Indonesian,English,Spanish,Chinese": 0.32386363636363635, "Indonesian,English,Spanish,Malay": 0.42045454545454547, "Indonesian,English,Spanish,Vietnamese": 0.42045454545454547, "Indonesian,English,Chinese,Malay": 0.3465909090909091, "Indonesian,English,Chinese,Vietnamese": 0.3125, "Indonesian,English,Malay,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Malay": 0.3465909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,Filipino,Chinese,Malay": 0.3125, "Indonesian,Filipino,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.3465909090909091, "Indonesian,Spanish,Chinese,Malay": 0.3068181818181818, "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.3977272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.3125, "English,Filipino,Spanish,Chinese": 0.32386363636363635, "English,Filipino,Spanish,Malay": 0.3465909090909091, "English,Filipino,Spanish,Vietnamese": 0.35795454545454547, "English,Filipino,Chinese,Malay": 0.3181818181818182, "English,Filipino,Chinese,Vietnamese": 0.29545454545454547, "English,Filipino,Malay,Vietnamese": 0.3409090909090909, "English,Spanish,Chinese,Malay": 0.3465909090909091, "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, "English,Spanish,Malay,Vietnamese": 0.4318181818181818, "English,Chinese,Malay,Vietnamese": 0.32386363636363635, "Filipino,Spanish,Chinese,Malay": 0.29545454545454547, "Filipino,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Filipino,Spanish,Malay,Vietnamese": 0.32954545454545453, "Filipino,Chinese,Malay,Vietnamese": 0.2840909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2556818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.30113636363636365, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.29545454545454547, "Indonesian,English,Filipino,Chinese,Malay": 0.2727272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.30113636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.2897727272727273, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3693181818181818, "Indonesian,English,Chinese,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese,Malay": 0.2784090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.30113636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.25 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.23863636363636365, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.26704545454545453, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.23295454545454544, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909 } }, "AC3_2": 0.522868616609141, "AC3_3": 0.44699701759051197, "AC3_4": 0.3928903327095309, "AC3_5": 0.3518703767071589, "AC3_6": 0.32050197815827963, "AC3_7": 0.29684328593074655 }, "prompt_5": { "overall_acc": 0.47159090909090917, "language_acc": { "Indonesian": 0.4318181818181818, "English": 0.5681818181818182, "Filipino": 0.4147727272727273, "Spanish": 0.4375, "Chinese": 0.5, "Malay": 0.4772727272727273, "Vietnamese": 0.4715909090909091 }, "consistency_score_2": 0.6025432900432899, "consistency_score_3": 0.4491883116883117, "consistency_score_4": 0.3680194805194806, "consistency_score_5": 0.31764069264069267, "consistency_score_6": 0.2824675324675325, "consistency_score_7": 0.2556818181818182, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6136363636363636, "Indonesian,Filipino": 0.5965909090909091, "Indonesian,Spanish": 0.6136363636363636, "Indonesian,Chinese": 0.5454545454545454, "Indonesian,Malay": 0.6875, "Indonesian,Vietnamese": 0.6136363636363636, "English,Filipino": 0.5625, "English,Spanish": 0.7102272727272727, "English,Chinese": 0.625, "English,Malay": 0.6477272727272727, "English,Vietnamese": 0.6590909090909091, "Filipino,Spanish": 0.5681818181818182, "Filipino,Chinese": 0.5454545454545454, "Filipino,Malay": 0.5397727272727273, "Filipino,Vietnamese": 0.5454545454545454, "Spanish,Chinese": 0.5795454545454546, "Spanish,Malay": 0.6306818181818182, "Spanish,Vietnamese": 0.6761363636363636, "Chinese,Malay": 0.5454545454545454, "Chinese,Vietnamese": 0.5397727272727273, "Malay,Vietnamese": 0.6079545454545454 }, "3_combine": { "Indonesian,English,Filipino": 0.44886363636363635, "Indonesian,English,Spanish": 0.5170454545454546, "Indonesian,English,Chinese": 0.4318181818181818, "Indonesian,English,Malay": 0.5056818181818182, "Indonesian,English,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Spanish": 0.4318181818181818, "Indonesian,Filipino,Chinese": 0.39204545454545453, "Indonesian,Filipino,Malay": 0.4602272727272727, "Indonesian,Filipino,Vietnamese": 0.4147727272727273, "Indonesian,Spanish,Chinese": 0.4147727272727273, "Indonesian,Spanish,Malay": 0.4943181818181818, "Indonesian,Spanish,Vietnamese": 0.48863636363636365, "Indonesian,Chinese,Malay": 0.4375, "Indonesian,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,Malay,Vietnamese": 0.5, "English,Filipino,Spanish": 0.45454545454545453, "English,Filipino,Chinese": 0.42613636363636365, "English,Filipino,Malay": 0.42613636363636365, "English,Filipino,Vietnamese": 0.4375, "English,Spanish,Chinese": 0.5, "English,Spanish,Malay": 0.5397727272727273, "English,Spanish,Vietnamese": 0.5511363636363636, "English,Chinese,Malay": 0.4431818181818182, "English,Chinese,Vietnamese": 0.4431818181818182, "English,Malay,Vietnamese": 0.48295454545454547, "Filipino,Spanish,Chinese": 0.4034090909090909, "Filipino,Spanish,Malay": 0.4147727272727273, "Filipino,Spanish,Vietnamese": 0.4431818181818182, "Filipino,Chinese,Malay": 0.3977272727272727, "Filipino,Chinese,Vietnamese": 0.3806818181818182, "Filipino,Malay,Vietnamese": 0.3977272727272727, "Spanish,Chinese,Malay": 0.4318181818181818, "Spanish,Chinese,Vietnamese": 0.4375, "Spanish,Malay,Vietnamese": 0.5, "Chinese,Malay,Vietnamese": 0.39204545454545453 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.3806818181818182, "Indonesian,English,Filipino,Chinese": 0.3465909090909091, "Indonesian,English,Filipino,Malay": 0.3806818181818182, "Indonesian,English,Filipino,Vietnamese": 0.35795454545454547, "Indonesian,English,Spanish,Chinese": 0.3806818181818182, "Indonesian,English,Spanish,Malay": 0.4375, "Indonesian,English,Spanish,Vietnamese": 0.4318181818181818, "Indonesian,English,Chinese,Malay": 0.3693181818181818, "Indonesian,English,Chinese,Vietnamese": 0.3522727272727273, "Indonesian,English,Malay,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.32954545454545453, "Indonesian,Filipino,Spanish,Malay": 0.3693181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.36363636363636365, "Indonesian,Filipino,Chinese,Malay": 0.3465909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.3068181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.3522727272727273, "Indonesian,Spanish,Chinese,Malay": 0.36363636363636365, "Indonesian,Spanish,Chinese,Vietnamese": 0.35795454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.4147727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.3465909090909091, "English,Filipino,Spanish,Chinese": 0.36363636363636365, "English,Filipino,Spanish,Malay": 0.3806818181818182, "English,Filipino,Spanish,Vietnamese": 0.38636363636363635, "English,Filipino,Chinese,Malay": 0.3409090909090909, "English,Filipino,Chinese,Vietnamese": 0.32954545454545453, "English,Filipino,Malay,Vietnamese": 0.36363636363636365, "English,Spanish,Chinese,Malay": 0.3977272727272727, "English,Spanish,Chinese,Vietnamese": 0.4034090909090909, "English,Spanish,Malay,Vietnamese": 0.44886363636363635, "English,Chinese,Malay,Vietnamese": 0.3522727272727273, "Filipino,Spanish,Chinese,Malay": 0.3409090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.3352272727272727, "Filipino,Spanish,Malay,Vietnamese": 0.36363636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.3125, "Spanish,Chinese,Malay,Vietnamese": 0.35795454545454547 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.3068181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.3352272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,English,Filipino,Chinese,Malay": 0.3068181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.32386363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.3352272727272727, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3352272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.3806818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.3125, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3068181818181818, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.3181818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.3125, "English,Filipino,Spanish,Malay,Vietnamese": 0.3409090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.3409090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.2840909090909091, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.30113636363636365, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 } }, "AC3_2": 0.529084611812433, "AC3_3": 0.4601170822682755, "AC3_4": 0.4134170884693576, "AC3_5": 0.3796007728505842, "AC3_6": 0.353312457140737, "AC3_7": 0.3315873579089522 } }, "sg_eval": { "prompt_1": { "accuracy": 0.5922330097087378 }, "prompt_2": { "accuracy": 0.6116504854368932 }, "prompt_3": { "accuracy": 0.6310679611650486 }, "prompt_4": { "accuracy": 0.6796116504854369 }, "prompt_5": { "accuracy": 0.6310679611650486 } }, "cn_eval": { "prompt_1": { "accuracy": 0.34285714285714286 }, "prompt_2": { "accuracy": 0.3142857142857143 }, "prompt_3": { "accuracy": 0.34285714285714286 }, "prompt_4": { "accuracy": 0.3619047619047619 }, "prompt_5": { "accuracy": 0.3523809523809524 } }, "us_eval": { "prompt_1": { "accuracy": 0.6915887850467289 }, "prompt_2": { "accuracy": 0.7009345794392523 }, "prompt_3": { "accuracy": 0.7476635514018691 }, "prompt_4": { "accuracy": 0.7663551401869159 }, "prompt_5": { "accuracy": 0.719626168224299 } }, "ph_eval": { "prompt_1": { "accuracy": 0.43, "category_acc": { "brand": 0.5, "demographics": 0.0, "biology": 0.4, "history": 0.4, "literature": 0.2, "politics": 0.6, "culture": 0.5, "film": 0.5, "law": 0.3, "geography": 0.7 } }, "prompt_2": { "accuracy": 0.43, "category_acc": { "brand": 0.4, "demographics": 0.4, "biology": 0.5, "history": 0.13333333333333333, "literature": 0.2, "politics": 0.9, "culture": 0.5, "film": 0.5, "law": 0.3, "geography": 0.6 } }, "prompt_3": { "accuracy": 0.44, "category_acc": { "brand": 0.4, "demographics": 0.2, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.7, "culture": 0.6, "film": 0.4, "law": 0.5, "geography": 0.6 } }, "prompt_4": { "accuracy": 0.47, "category_acc": { "brand": 0.6, "demographics": 0.2, "biology": 0.4, "history": 0.3333333333333333, "literature": 0.3, "politics": 0.8, "culture": 0.4, "film": 0.5, "law": 0.4, "geography": 0.7 } }, "prompt_5": { "accuracy": 0.42, "category_acc": { "brand": 0.4, "demographics": 0.2, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.7, "culture": 0.5, "film": 0.4, "law": 0.4, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.25083986933748753 }, "prompt_2": { "bleu_score": 0.23347550471383358 }, "prompt_3": { "bleu_score": 0.22918830819501845 }, "prompt_4": { "bleu_score": 0.21613737202527156 }, "prompt_5": { "bleu_score": 0.21084748299774206 } }, "indommlu": { "prompt_1": { "accuracy": 0.42626343547633355, "category_acc": { "History": 0.3413654618473896, "Geography": 0.35306122448979593, "Lampungic": 0.36054421768707484, "Social science": 0.672787979966611, "Balinese": 0.29936305732484075, "Makassarese": 0.3817204301075269, "Banjarese": 0.3541666666666667, "Chemistry": 0.25401459854014596, "Biology": 0.357396449704142, "Science": 0.5964912280701754, "Christian religion": 0.48258706467661694, "Art": 0.519134775374376, "Islam religion": 0.48933143669985774, "Hindu religion": 0.44, "Madurese": 0.29152542372881357, "Sport": 0.40540540540540543, "Indonesian language": 0.5168119551681195, "Physics": 0.41414141414141414, "Minangkabau culture": 0.35175879396984927, "Dayak language": 0.3119266055045872, "Sociology": 0.3326612903225806, "Economy": 0.3319672131147541, "Sundanese": 0.3197925669835782, "Javanese": 0.31350806451612906, "Civic education": 0.4678111587982833 } }, "prompt_2": { "accuracy": 0.4431537485813472, "category_acc": { "History": 0.3634538152610442, "Geography": 0.35918367346938773, "Lampungic": 0.40816326530612246, "Social science": 0.7212020033388982, "Balinese": 0.3078556263269639, "Makassarese": 0.3978494623655914, "Banjarese": 0.3541666666666667, "Chemistry": 0.29635036496350364, "Biology": 0.40236686390532544, "Science": 0.6006191950464397, "Christian religion": 0.4925373134328358, "Art": 0.5224625623960066, "Islam religion": 0.5120910384068279, "Hindu religion": 0.43333333333333335, "Madurese": 0.3016949152542373, "Sport": 0.39864864864864863, "Indonesian language": 0.5255292652552926, "Physics": 0.4222222222222222, "Minangkabau culture": 0.32160804020100503, "Dayak language": 0.28440366972477066, "Sociology": 0.37701612903225806, "Economy": 0.35040983606557374, "Sundanese": 0.33967156439066554, "Javanese": 0.3074596774193548, "Civic education": 0.5150214592274678 } }, "prompt_3": { "accuracy": 0.4425529074036985, "category_acc": { "History": 0.3714859437751004, "Geography": 0.36122448979591837, "Lampungic": 0.41496598639455784, "Social science": 0.6978297161936561, "Balinese": 0.29936305732484075, "Makassarese": 0.42473118279569894, "Banjarese": 0.3680555555555556, "Chemistry": 0.2686131386861314, "Biology": 0.3940828402366864, "Science": 0.6057791537667698, "Christian religion": 0.48756218905472637, "Art": 0.5291181364392679, "Islam religion": 0.5277382645803699, "Hindu religion": 0.4666666666666667, "Madurese": 0.2847457627118644, "Sport": 0.44594594594594594, "Indonesian language": 0.5214819427148194, "Physics": 0.42424242424242425, "Minangkabau culture": 0.3316582914572864, "Dayak language": 0.29357798165137616, "Sociology": 0.36088709677419356, "Economy": 0.3831967213114754, "Sundanese": 0.35868625756266204, "Javanese": 0.3034274193548387, "Civic education": 0.48497854077253216 } }, "prompt_4": { "accuracy": 0.449295680619534, "category_acc": { "History": 0.39357429718875503, "Geography": 0.39183673469387753, "Lampungic": 0.35374149659863946, "Social science": 0.7178631051752922, "Balinese": 0.3057324840764331, "Makassarese": 0.3548387096774194, "Banjarese": 0.3472222222222222, "Chemistry": 0.24963503649635035, "Biology": 0.41183431952662725, "Science": 0.6191950464396285, "Christian religion": 0.48258706467661694, "Art": 0.5124792013311148, "Islam religion": 0.5135135135135135, "Hindu religion": 0.4866666666666667, "Madurese": 0.2677966101694915, "Sport": 0.4391891891891892, "Indonesian language": 0.5317559153175592, "Physics": 0.4202020202020202, "Minangkabau culture": 0.3316582914572864, "Dayak language": 0.3486238532110092, "Sociology": 0.4012096774193548, "Economy": 0.39959016393442626, "Sundanese": 0.35868625756266204, "Javanese": 0.3155241935483871, "Civic education": 0.5092989985693849 } }, "prompt_5": { "accuracy": 0.43834701916015756, "category_acc": { "History": 0.3895582329317269, "Geography": 0.35918367346938773, "Lampungic": 0.4013605442176871, "Social science": 0.67779632721202, "Balinese": 0.32908704883227174, "Makassarese": 0.3279569892473118, "Banjarese": 0.3263888888888889, "Chemistry": 0.27445255474452557, "Biology": 0.38698224852071006, "Science": 0.6160990712074303, "Christian religion": 0.472636815920398, "Art": 0.502495840266223, "Islam religion": 0.5106685633001422, "Hindu religion": 0.48, "Madurese": 0.26101694915254237, "Sport": 0.41216216216216217, "Indonesian language": 0.5155666251556662, "Physics": 0.41414141414141414, "Minangkabau culture": 0.32663316582914576, "Dayak language": 0.3486238532110092, "Sociology": 0.3629032258064516, "Economy": 0.39549180327868855, "Sundanese": 0.3474503025064823, "Javanese": 0.29435483870967744, "Civic education": 0.5135908440629471 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.33349561081539403 }, "prompt_2": { "bleu_score": 0.34924350312022184 }, "prompt_3": { "bleu_score": 0.34642710360033174 }, "prompt_4": { "bleu_score": 0.351335394546623 }, "prompt_5": { "bleu_score": 0.33902183478630804 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.28077818415977956 }, "prompt_2": { "bleu_score": 0.29263312173317624 }, "prompt_3": { "bleu_score": 0.29003004114772607 }, "prompt_4": { "bleu_score": 0.2937442705094875 }, "prompt_5": { "bleu_score": 0.28097633581136966 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.20874149964374597 }, "prompt_2": { "bleu_score": 0.2128290297582051 }, "prompt_3": { "bleu_score": 0.21013614813602086 }, "prompt_4": { "bleu_score": 0.2155940961935285 }, "prompt_5": { "bleu_score": 0.20770689743003645 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.355017126213193 }, "prompt_2": { "bleu_score": 0.3603933291111741 }, "prompt_3": { "bleu_score": 0.3594937927110909 }, "prompt_4": { "bleu_score": 0.3585242382452043 }, "prompt_5": { "bleu_score": 0.3510843144050436 } }, "mmlu": { "prompt_1": { "accuracy": 0.6231038506417736 }, "prompt_2": { "accuracy": 0.6242707117852976 }, "prompt_3": { "accuracy": 0.617269544924154 }, "prompt_4": { "accuracy": 0.6231038506417736 }, "prompt_5": { "accuracy": 0.6196032672112018 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.6007150518412585, "category_acc": { "high_school_european_history": 0.7439024390243902, "business_ethics": 0.5656565656565656, "clinical_knowledge": 0.6704545454545454, "medical_genetics": 0.6363636363636364, "high_school_us_history": 0.8029556650246306, "high_school_physics": 0.36666666666666664, "high_school_world_history": 0.7966101694915254, "virology": 0.4727272727272727, "high_school_microeconomics": 0.6497890295358649, "econometrics": 0.415929203539823, "college_computer_science": 0.5151515151515151, "high_school_biology": 0.7572815533980582, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.45195729537366547, "philosophy": 0.6419354838709678, "professional_medicine": 0.6568265682656826, "nutrition": 0.6622950819672131, "global_facts": 0.26262626262626265, "machine_learning": 0.36936936936936937, "security_studies": 0.7090163934426229, "public_relations": 0.6146788990825688, "professional_psychology": 0.6202945990180033, "prehistory": 0.6873065015479877, "anatomy": 0.5597014925373134, "human_sexuality": 0.6461538461538462, "college_medicine": 0.622093023255814, "high_school_government_and_politics": 0.8541666666666666, "college_chemistry": 0.5050505050505051, "logical_fallacies": 0.7345679012345679, "high_school_geography": 0.7868020304568528, "elementary_mathematics": 0.5994694960212201, "human_aging": 0.6486486486486487, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.8106617647058824, "formal_logic": 0.432, "high_school_statistics": 0.42790697674418604, "international_law": 0.7833333333333333, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.6868686868686869, "conceptual_physics": 0.5299145299145299, "miscellaneous": 0.8081841432225064, "high_school_chemistry": 0.42574257425742573, "marketing": 0.8412017167381974, "professional_law": 0.4435746901500326, "management": 0.7745098039215687, "college_physics": 0.3465346534653465, "jurisprudence": 0.7383177570093458, "world_religions": 0.8235294117647058, "sociology": 0.815, "us_foreign_policy": 0.8383838383838383, "high_school_macroeconomics": 0.6143958868894601, "computer_security": 0.6666666666666666, "moral_scenarios": 0.3187919463087248, "moral_disputes": 0.663768115942029, "electrical_engineering": 0.5416666666666666, "astronomy": 0.6490066225165563, "college_biology": 0.7482517482517482 } }, "prompt_2": { "accuracy": 0.6010010725777619, "category_acc": { "high_school_european_history": 0.725609756097561, "business_ethics": 0.5656565656565656, "clinical_knowledge": 0.6742424242424242, "medical_genetics": 0.6565656565656566, "high_school_us_history": 0.8078817733990148, "high_school_physics": 0.32, "high_school_world_history": 0.809322033898305, "virology": 0.4666666666666667, "high_school_microeconomics": 0.6540084388185654, "econometrics": 0.4247787610619469, "college_computer_science": 0.48484848484848486, "high_school_biology": 0.7637540453074434, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.41637010676156583, "philosophy": 0.6516129032258065, "professional_medicine": 0.6014760147601476, "nutrition": 0.6524590163934426, "global_facts": 0.31313131313131315, "machine_learning": 0.43243243243243246, "security_studies": 0.6844262295081968, "public_relations": 0.5871559633027523, "professional_psychology": 0.6121112929623568, "prehistory": 0.6996904024767802, "anatomy": 0.5597014925373134, "human_sexuality": 0.6461538461538462, "college_medicine": 0.6162790697674418, "high_school_government_and_politics": 0.8489583333333334, "college_chemistry": 0.48484848484848486, "logical_fallacies": 0.7345679012345679, "high_school_geography": 0.7969543147208121, "elementary_mathematics": 0.5862068965517241, "human_aging": 0.6621621621621622, "college_mathematics": 0.3333333333333333, "high_school_psychology": 0.8198529411764706, "formal_logic": 0.4, "high_school_statistics": 0.4232558139534884, "international_law": 0.7833333333333333, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.6565656565656566, "conceptual_physics": 0.5256410256410257, "miscellaneous": 0.8107416879795396, "high_school_chemistry": 0.44554455445544555, "marketing": 0.8454935622317596, "professional_law": 0.44031311154598823, "management": 0.803921568627451, "college_physics": 0.37623762376237624, "jurisprudence": 0.7383177570093458, "world_religions": 0.8058823529411765, "sociology": 0.815, "us_foreign_policy": 0.8080808080808081, "high_school_macroeconomics": 0.6426735218508998, "computer_security": 0.7070707070707071, "moral_scenarios": 0.3434004474272931, "moral_disputes": 0.663768115942029, "electrical_engineering": 0.5763888888888888, "astronomy": 0.6291390728476821, "college_biology": 0.7132867132867133 } }, "prompt_3": { "accuracy": 0.6034322488380408, "category_acc": { "high_school_european_history": 0.7378048780487805, "business_ethics": 0.5757575757575758, "clinical_knowledge": 0.6818181818181818, "medical_genetics": 0.6363636363636364, "high_school_us_history": 0.7980295566502463, "high_school_physics": 0.35333333333333333, "high_school_world_history": 0.8050847457627118, "virology": 0.4727272727272727, "high_school_microeconomics": 0.6455696202531646, "econometrics": 0.39823008849557523, "college_computer_science": 0.48484848484848486, "high_school_biology": 0.7475728155339806, "abstract_algebra": 0.25252525252525254, "professional_accounting": 0.4306049822064057, "philosophy": 0.6709677419354839, "professional_medicine": 0.6273062730627307, "nutrition": 0.6622950819672131, "global_facts": 0.3838383838383838, "machine_learning": 0.43243243243243246, "security_studies": 0.7172131147540983, "public_relations": 0.6146788990825688, "professional_psychology": 0.6219312602291326, "prehistory": 0.6811145510835913, "anatomy": 0.5373134328358209, "human_sexuality": 0.6538461538461539, "college_medicine": 0.6104651162790697, "high_school_government_and_politics": 0.859375, "college_chemistry": 0.42424242424242425, "logical_fallacies": 0.7222222222222222, "high_school_geography": 0.7868020304568528, "elementary_mathematics": 0.5888594164456233, "human_aging": 0.6576576576576577, "college_mathematics": 0.41414141414141414, "high_school_psychology": 0.8180147058823529, "formal_logic": 0.4, "high_school_statistics": 0.4558139534883721, "international_law": 0.7916666666666666, "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.6868686868686869, "conceptual_physics": 0.5213675213675214, "miscellaneous": 0.8158567774936062, "high_school_chemistry": 0.4504950495049505, "marketing": 0.8412017167381974, "professional_law": 0.441617742987606, "management": 0.7941176470588235, "college_physics": 0.3564356435643564, "jurisprudence": 0.7663551401869159, "world_religions": 0.8058823529411765, "sociology": 0.805, "us_foreign_policy": 0.8484848484848485, "high_school_macroeconomics": 0.6401028277634961, "computer_security": 0.6666666666666666, "moral_scenarios": 0.32997762863534674, "moral_disputes": 0.6608695652173913, "electrical_engineering": 0.5486111111111112, "astronomy": 0.6357615894039735, "college_biology": 0.7202797202797203 } }, "prompt_4": { "accuracy": 0.6067929924919556, "category_acc": { "high_school_european_history": 0.7317073170731707, "business_ethics": 0.5858585858585859, "clinical_knowledge": 0.678030303030303, "medical_genetics": 0.6363636363636364, "high_school_us_history": 0.8029556650246306, "high_school_physics": 0.32666666666666666, "high_school_world_history": 0.7923728813559322, "virology": 0.4666666666666667, "high_school_microeconomics": 0.6666666666666666, "econometrics": 0.40707964601769914, "college_computer_science": 0.5454545454545454, "high_school_biology": 0.7637540453074434, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.45195729537366547, "philosophy": 0.6645161290322581, "professional_medicine": 0.6199261992619927, "nutrition": 0.6655737704918033, "global_facts": 0.3838383838383838, "machine_learning": 0.3963963963963964, "security_studies": 0.6967213114754098, "public_relations": 0.6330275229357798, "professional_psychology": 0.6219312602291326, "prehistory": 0.6965944272445821, "anatomy": 0.5746268656716418, "human_sexuality": 0.6923076923076923, "college_medicine": 0.6162790697674418, "high_school_government_and_politics": 0.8489583333333334, "college_chemistry": 0.5151515151515151, "logical_fallacies": 0.7098765432098766, "high_school_geography": 0.7969543147208121, "elementary_mathematics": 0.5888594164456233, "human_aging": 0.6621621621621622, "college_mathematics": 0.35353535353535354, "high_school_psychology": 0.8161764705882353, "formal_logic": 0.392, "high_school_statistics": 0.48372093023255813, "international_law": 0.7833333333333333, "high_school_mathematics": 0.3048327137546468, "high_school_computer_science": 0.696969696969697, "conceptual_physics": 0.5213675213675214, "miscellaneous": 0.819693094629156, "high_school_chemistry": 0.400990099009901, "marketing": 0.8540772532188842, "professional_law": 0.4461839530332681, "management": 0.7745098039215687, "college_physics": 0.36633663366336633, "jurisprudence": 0.7289719626168224, "world_religions": 0.7941176470588235, "sociology": 0.82, "us_foreign_policy": 0.8181818181818182, "high_school_macroeconomics": 0.6452442159383034, "computer_security": 0.6767676767676768, "moral_scenarios": 0.3288590604026846, "moral_disputes": 0.6840579710144927, "electrical_engineering": 0.5486111111111112, "astronomy": 0.6490066225165563, "college_biology": 0.7412587412587412 } }, "prompt_5": { "accuracy": 0.5982838755809796, "category_acc": { "high_school_european_history": 0.7073170731707317, "business_ethics": 0.5454545454545454, "clinical_knowledge": 0.6553030303030303, "medical_genetics": 0.6464646464646465, "high_school_us_history": 0.7635467980295566, "high_school_physics": 0.36666666666666664, "high_school_world_history": 0.8050847457627118, "virology": 0.48484848484848486, "high_school_microeconomics": 0.6497890295358649, "econometrics": 0.36283185840707965, "college_computer_science": 0.5353535353535354, "high_school_biology": 0.7411003236245954, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.4092526690391459, "philosophy": 0.6483870967741936, "professional_medicine": 0.5756457564575646, "nutrition": 0.6557377049180327, "global_facts": 0.3939393939393939, "machine_learning": 0.3963963963963964, "security_studies": 0.7049180327868853, "public_relations": 0.5871559633027523, "professional_psychology": 0.6235679214402619, "prehistory": 0.6873065015479877, "anatomy": 0.5373134328358209, "human_sexuality": 0.6615384615384615, "college_medicine": 0.6337209302325582, "high_school_government_and_politics": 0.859375, "college_chemistry": 0.47474747474747475, "logical_fallacies": 0.7283950617283951, "high_school_geography": 0.7918781725888325, "elementary_mathematics": 0.583554376657825, "human_aging": 0.6441441441441441, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.8180147058823529, "formal_logic": 0.376, "high_school_statistics": 0.4604651162790698, "international_law": 0.7833333333333333, "high_school_mathematics": 0.2788104089219331, "high_school_computer_science": 0.6767676767676768, "conceptual_physics": 0.5213675213675214, "miscellaneous": 0.8107416879795396, "high_school_chemistry": 0.44554455445544555, "marketing": 0.8326180257510729, "professional_law": 0.4409654272667971, "management": 0.7745098039215687, "college_physics": 0.40594059405940597, "jurisprudence": 0.7102803738317757, "world_religions": 0.8352941176470589, "sociology": 0.825, "us_foreign_policy": 0.8383838383838383, "high_school_macroeconomics": 0.6195372750642674, "computer_security": 0.6666666666666666, "moral_scenarios": 0.3232662192393736, "moral_disputes": 0.663768115942029, "electrical_engineering": 0.5486111111111112, "astronomy": 0.6490066225165563, "college_biology": 0.7062937062937062 } } }, "c_eval": { "prompt_1": { "accuracy": 0.42867756315007427 }, "prompt_2": { "accuracy": 0.43684992570579495 }, "prompt_3": { "accuracy": 0.462109955423477 }, "prompt_4": { "accuracy": 0.4182763744427935 }, "prompt_5": { "accuracy": 0.424962852897474 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.4396014943960149, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.5833333333333334, "computer_architecture": 0.5, "college_programming": 0.6666666666666666, "college_physics": 0.25, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.47619047619047616, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.6923076923076923, "middle_school_physics": 0.5, "middle_school_chemistry": 0.48, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.48333333333333334, "business_administration": 0.47368421052631576, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.5862068965517241, "education_science": 0.5, "teacher_qualification": 0.673469387755102, "high_school_politics": 0.5, "high_school_geography": 0.5416666666666666, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.5294117647058824, "modern_chinese_history": 0.2857142857142857, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.4074074074074074, "law": 0.3448275862068966, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.25, "high_school_chinese": 0.375, "high_school_history": 0.6, "middle_school_history": 0.5185185185185185, "civil_servant": 0.4230769230769231, "sports_science": 0.4166666666666667, "plant_protection": 0.5925925925925926, "basic_medicine": 0.5, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.49019607843137253, "accountant": 0.3888888888888889, "fire_engineer": 0.16666666666666666, "environmental_impact_assessment_engineer": 0.5833333333333334, "tax_accountant": 0.3148148148148148, "physician": 0.46296296296296297 } }, "prompt_2": { "accuracy": 0.44956413449564137, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.5833333333333334, "computer_architecture": 0.46153846153846156, "college_programming": 0.5238095238095238, "college_physics": 0.20833333333333334, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.5238095238095238, "electrical_engineer": 0.5476190476190477, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.25, "high_school_chemistry": 0.375, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.25, "middle_school_biology": 0.7692307692307693, "middle_school_physics": 0.5416666666666666, "middle_school_chemistry": 0.48, "veterinary_medicine": 0.39285714285714285, "college_economics": 0.4166666666666667, "business_administration": 0.39473684210526316, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.5517241379310345, "education_science": 0.4411764705882353, "teacher_qualification": 0.673469387755102, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.5416666666666666, "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.35714285714285715, "ideological_and_moral_cultivation": 0.625, "logic": 0.5185185185185185, "law": 0.2413793103448276, "chinese_language_and_literature": 0.5, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.25, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.56, "middle_school_history": 0.5555555555555556, "civil_servant": 0.4423076923076923, "sports_science": 0.375, "plant_protection": 0.5555555555555556, "basic_medicine": 0.5833333333333334, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.5686274509803921, "accountant": 0.3888888888888889, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.5833333333333334, "tax_accountant": 0.48148148148148145, "physician": 0.4444444444444444 } }, "prompt_3": { "accuracy": 0.4396014943960149, "category_acc": { "computer_network": 0.4166666666666667, "operating_system": 0.625, "computer_architecture": 0.6538461538461539, "college_programming": 0.5714285714285714, "college_physics": 0.375, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.47619047619047616, "metrology_engineer": 0.3103448275862069, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.375, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.6153846153846154, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.4166666666666667, "business_administration": 0.47368421052631576, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.6206896551724138, "education_science": 0.5, "teacher_qualification": 0.7142857142857143, "high_school_politics": 0.4583333333333333, "high_school_geography": 0.5416666666666666, "middle_school_politics": 0.6538461538461539, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.39285714285714285, "ideological_and_moral_cultivation": 0.6666666666666666, "logic": 0.3333333333333333, "law": 0.41379310344827586, "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.38235294117647056, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.375, "high_school_history": 0.6, "middle_school_history": 0.4444444444444444, "civil_servant": 0.5, "sports_science": 0.3333333333333333, "plant_protection": 0.5925925925925926, "basic_medicine": 0.5416666666666666, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.5098039215686274, "accountant": 0.3148148148148148, "fire_engineer": 0.3333333333333333, "environmental_impact_assessment_engineer": 0.5833333333333334, "tax_accountant": 0.4074074074074074, "physician": 0.4444444444444444 } }, "prompt_4": { "accuracy": 0.4202988792029888, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5833333333333334, "computer_architecture": 0.5, "college_programming": 0.47619047619047616, "college_physics": 0.125, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.25, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.5238095238095238, "electrical_engineer": 0.3333333333333333, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.08695652173913043, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.25, "middle_school_biology": 0.6923076923076923, "middle_school_physics": 0.5, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.25, "college_economics": 0.4, "business_administration": 0.42105263157894735, "marxism": 0.5, "mao_zedong_thought": 0.5172413793103449, "education_science": 0.5, "teacher_qualification": 0.6530612244897959, "high_school_politics": 0.4583333333333333, "high_school_geography": 0.5833333333333334, "middle_school_politics": 0.5769230769230769, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.39285714285714285, "ideological_and_moral_cultivation": 0.5, "logic": 0.4074074074074074, "law": 0.3103448275862069, "chinese_language_and_literature": 0.35714285714285715, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.52, "middle_school_history": 0.5925925925925926, "civil_servant": 0.40384615384615385, "sports_science": 0.375, "plant_protection": 0.5185185185185185, "basic_medicine": 0.6666666666666666, "clinical_medicine": 0.48148148148148145, "urban_and_rural_planner": 0.5294117647058824, "accountant": 0.3333333333333333, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.4722222222222222, "tax_accountant": 0.3888888888888889, "physician": 0.4444444444444444 } }, "prompt_5": { "accuracy": 0.42092154420921546, "category_acc": { "computer_network": 0.5, "operating_system": 0.625, "computer_architecture": 0.46153846153846156, "college_programming": 0.5714285714285714, "college_physics": 0.20833333333333334, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.375, "middle_school_biology": 0.5769230769230769, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.38333333333333336, "business_administration": 0.39473684210526316, "marxism": 0.375, "mao_zedong_thought": 0.6206896551724138, "education_science": 0.47058823529411764, "teacher_qualification": 0.673469387755102, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.5, "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.35714285714285715, "ideological_and_moral_cultivation": 0.7083333333333334, "logic": 0.4074074074074074, "law": 0.4827586206896552, "chinese_language_and_literature": 0.42857142857142855, "art_studies": 0.3157894736842105, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.375, "high_school_history": 0.36, "middle_school_history": 0.5185185185185185, "civil_servant": 0.38461538461538464, "sports_science": 0.375, "plant_protection": 0.6296296296296297, "basic_medicine": 0.4583333333333333, "clinical_medicine": 0.48148148148148145, "urban_and_rural_planner": 0.39215686274509803, "accountant": 0.3148148148148148, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.4444444444444444, "tax_accountant": 0.3333333333333333, "physician": 0.42592592592592593 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.4731182795698925 }, "prompt_2": { "accuracy": 0.5053763440860215 }, "prompt_3": { "accuracy": 0.4982078853046595 }, "prompt_4": { "accuracy": 0.4659498207885305 }, "prompt_5": { "accuracy": 0.46236559139784944 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.442756000690727, "category_acc": { "agronomy": 0.4319526627218935, "anatomy": 0.2972972972972973, "ancient_chinese": 0.2926829268292683, "arts": 0.5, "astronomy": 0.3939393939393939, "business_ethics": 0.5119617224880383, "chinese_civil_service_exam": 0.3625, "chinese_driving_rule": 0.5725190839694656, "chinese_food_culture": 0.33088235294117646, "chinese_foreign_policy": 0.4766355140186916, "chinese_history": 0.43653250773993807, "chinese_literature": 0.3137254901960784, "chinese_teacher_qualification": 0.553072625698324, "clinical_knowledge": 0.33755274261603374, "college_actuarial_science": 0.2358490566037736, "college_education": 0.5420560747663551, "college_engineering_hydrology": 0.5094339622641509, "college_law": 0.25925925925925924, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.4056603773584906, "college_medicine": 0.34798534798534797, "computer_science": 0.5784313725490197, "computer_security": 0.5730994152046783, "conceptual_physics": 0.4421768707482993, "construction_project_management": 0.4028776978417266, "economics": 0.5660377358490566, "education": 0.5030674846625767, "electrical_engineering": 0.4476744186046512, "elementary_chinese": 0.2857142857142857, "elementary_commonsense": 0.45454545454545453, "elementary_information_and_technology": 0.6974789915966386, "elementary_mathematics": 0.391304347826087, "ethnology": 0.4, "food_science": 0.4755244755244755, "genetics": 0.39204545454545453, "global_facts": 0.5369127516778524, "high_school_biology": 0.3431952662721893, "high_school_chemistry": 0.25, "high_school_geography": 0.3728813559322034, "high_school_mathematics": 0.2926829268292683, "high_school_physics": 0.38181818181818183, "high_school_politics": 0.34965034965034963, "human_sexuality": 0.5873015873015873, "international_law": 0.3621621621621622, "journalism": 0.5232558139534884, "jurisprudence": 0.46958637469586373, "legal_and_moral_basis": 0.705607476635514, "logical": 0.42276422764227645, "machine_learning": 0.4426229508196721, "management": 0.5857142857142857, "marketing": 0.5833333333333334, "marxist_theory": 0.5185185185185185, "modern_chinese": 0.3448275862068966, "nutrition": 0.4896551724137931, "philosophy": 0.41904761904761906, "professional_accounting": 0.4742857142857143, "professional_law": 0.36018957345971564, "professional_medicine": 0.324468085106383, "professional_psychology": 0.46551724137931033, "public_relations": 0.47701149425287354, "security_study": 0.5333333333333333, "sociology": 0.5309734513274337, "sports_science": 0.48484848484848486, "traditional_chinese_medicine": 0.372972972972973, "virology": 0.48520710059171596, "world_history": 0.45962732919254656, "world_religions": 0.54375 } }, "prompt_2": { "accuracy": 0.44620963564151267, "category_acc": { "agronomy": 0.4556213017751479, "anatomy": 0.30405405405405406, "ancient_chinese": 0.27439024390243905, "arts": 0.50625, "astronomy": 0.3393939393939394, "business_ethics": 0.5454545454545454, "chinese_civil_service_exam": 0.33125, "chinese_driving_rule": 0.6183206106870229, "chinese_food_culture": 0.3161764705882353, "chinese_foreign_policy": 0.4485981308411215, "chinese_history": 0.44891640866873067, "chinese_literature": 0.3333333333333333, "chinese_teacher_qualification": 0.5307262569832403, "clinical_knowledge": 0.350210970464135, "college_actuarial_science": 0.25471698113207547, "college_education": 0.5887850467289719, "college_engineering_hydrology": 0.5, "college_law": 0.32407407407407407, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.29245283018867924, "college_medicine": 0.3772893772893773, "computer_science": 0.5735294117647058, "computer_security": 0.6081871345029239, "conceptual_physics": 0.38095238095238093, "construction_project_management": 0.4028776978417266, "economics": 0.559748427672956, "education": 0.50920245398773, "electrical_engineering": 0.45930232558139533, "elementary_chinese": 0.29365079365079366, "elementary_commonsense": 0.4494949494949495, "elementary_information_and_technology": 0.6974789915966386, "elementary_mathematics": 0.3652173913043478, "ethnology": 0.4074074074074074, "food_science": 0.4755244755244755, "genetics": 0.3977272727272727, "global_facts": 0.5302013422818792, "high_school_biology": 0.3136094674556213, "high_school_chemistry": 0.2878787878787879, "high_school_geography": 0.3305084745762712, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.43636363636363634, "high_school_politics": 0.42657342657342656, "human_sexuality": 0.5634920634920635, "international_law": 0.3891891891891892, "journalism": 0.4883720930232558, "jurisprudence": 0.46958637469586373, "legal_and_moral_basis": 0.7523364485981309, "logical": 0.4634146341463415, "machine_learning": 0.45901639344262296, "management": 0.5619047619047619, "marketing": 0.6055555555555555, "marxist_theory": 0.5079365079365079, "modern_chinese": 0.35344827586206895, "nutrition": 0.503448275862069, "philosophy": 0.45714285714285713, "professional_accounting": 0.5085714285714286, "professional_law": 0.3696682464454976, "professional_medicine": 0.31648936170212766, "professional_psychology": 0.4698275862068966, "public_relations": 0.4942528735632184, "security_study": 0.5259259259259259, "sociology": 0.4778761061946903, "sports_science": 0.48484848484848486, "traditional_chinese_medicine": 0.3783783783783784, "virology": 0.5266272189349113, "world_history": 0.5031055900621118, "world_religions": 0.525 } }, "prompt_3": { "accuracy": 0.439129683992402, "category_acc": { "agronomy": 0.4319526627218935, "anatomy": 0.3108108108108108, "ancient_chinese": 0.2682926829268293, "arts": 0.4875, "astronomy": 0.34545454545454546, "business_ethics": 0.5550239234449761, "chinese_civil_service_exam": 0.36875, "chinese_driving_rule": 0.5725190839694656, "chinese_food_culture": 0.3235294117647059, "chinese_foreign_policy": 0.5420560747663551, "chinese_history": 0.43962848297213625, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.4972067039106145, "clinical_knowledge": 0.34177215189873417, "college_actuarial_science": 0.29245283018867924, "college_education": 0.5794392523364486, "college_engineering_hydrology": 0.49056603773584906, "college_law": 0.37037037037037035, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.3113207547169811, "college_medicine": 0.3772893772893773, "computer_science": 0.5784313725490197, "computer_security": 0.5789473684210527, "conceptual_physics": 0.3877551020408163, "construction_project_management": 0.43884892086330934, "economics": 0.5534591194968553, "education": 0.48466257668711654, "electrical_engineering": 0.45930232558139533, "elementary_chinese": 0.2976190476190476, "elementary_commonsense": 0.42424242424242425, "elementary_information_and_technology": 0.6974789915966386, "elementary_mathematics": 0.3521739130434783, "ethnology": 0.4222222222222222, "food_science": 0.44755244755244755, "genetics": 0.39204545454545453, "global_facts": 0.5100671140939598, "high_school_biology": 0.35502958579881655, "high_school_chemistry": 0.21212121212121213, "high_school_geography": 0.3983050847457627, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.3181818181818182, "high_school_politics": 0.38461538461538464, "human_sexuality": 0.5396825396825397, "international_law": 0.3945945945945946, "journalism": 0.4883720930232558, "jurisprudence": 0.4720194647201946, "legal_and_moral_basis": 0.6682242990654206, "logical": 0.45528455284552843, "machine_learning": 0.4426229508196721, "management": 0.5761904761904761, "marketing": 0.5722222222222222, "marxist_theory": 0.5238095238095238, "modern_chinese": 0.29310344827586204, "nutrition": 0.5103448275862069, "philosophy": 0.5238095238095238, "professional_accounting": 0.45714285714285713, "professional_law": 0.35071090047393366, "professional_medicine": 0.3058510638297872, "professional_psychology": 0.46120689655172414, "public_relations": 0.5172413793103449, "security_study": 0.5185185185185185, "sociology": 0.5176991150442478, "sports_science": 0.4121212121212121, "traditional_chinese_medicine": 0.33513513513513515, "virology": 0.5088757396449705, "world_history": 0.546583850931677, "world_religions": 0.5125 } }, "prompt_4": { "accuracy": 0.4350716629252288, "category_acc": { "agronomy": 0.4260355029585799, "anatomy": 0.30405405405405406, "ancient_chinese": 0.2926829268292683, "arts": 0.4625, "astronomy": 0.296969696969697, "business_ethics": 0.5215311004784688, "chinese_civil_service_exam": 0.35625, "chinese_driving_rule": 0.5648854961832062, "chinese_food_culture": 0.35294117647058826, "chinese_foreign_policy": 0.514018691588785, "chinese_history": 0.42724458204334365, "chinese_literature": 0.3382352941176471, "chinese_teacher_qualification": 0.5083798882681564, "clinical_knowledge": 0.3080168776371308, "college_actuarial_science": 0.20754716981132076, "college_education": 0.5046728971962616, "college_engineering_hydrology": 0.5, "college_law": 0.3611111111111111, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.330188679245283, "college_medicine": 0.3516483516483517, "computer_science": 0.5343137254901961, "computer_security": 0.5614035087719298, "conceptual_physics": 0.41496598639455784, "construction_project_management": 0.45323741007194246, "economics": 0.5094339622641509, "education": 0.4662576687116564, "electrical_engineering": 0.43023255813953487, "elementary_chinese": 0.2976190476190476, "elementary_commonsense": 0.4292929292929293, "elementary_information_and_technology": 0.680672268907563, "elementary_mathematics": 0.39565217391304347, "ethnology": 0.4, "food_science": 0.5104895104895105, "genetics": 0.38636363636363635, "global_facts": 0.5033557046979866, "high_school_biology": 0.33136094674556216, "high_school_chemistry": 0.24242424242424243, "high_school_geography": 0.3898305084745763, "high_school_mathematics": 0.3170731707317073, "high_school_physics": 0.3181818181818182, "high_school_politics": 0.40559440559440557, "human_sexuality": 0.5714285714285714, "international_law": 0.3783783783783784, "journalism": 0.5, "jurisprudence": 0.49148418491484186, "legal_and_moral_basis": 0.719626168224299, "logical": 0.44715447154471544, "machine_learning": 0.4672131147540984, "management": 0.5619047619047619, "marketing": 0.5944444444444444, "marxist_theory": 0.5343915343915344, "modern_chinese": 0.27586206896551724, "nutrition": 0.4689655172413793, "philosophy": 0.44761904761904764, "professional_accounting": 0.5028571428571429, "professional_law": 0.3459715639810427, "professional_medicine": 0.31648936170212766, "professional_psychology": 0.4482758620689655, "public_relations": 0.4827586206896552, "security_study": 0.45185185185185184, "sociology": 0.4690265486725664, "sports_science": 0.4484848484848485, "traditional_chinese_medicine": 0.34594594594594597, "virology": 0.4911242603550296, "world_history": 0.4906832298136646, "world_religions": 0.5125 } }, "prompt_5": { "accuracy": 0.4328268002072181, "category_acc": { "agronomy": 0.40236686390532544, "anatomy": 0.30405405405405406, "ancient_chinese": 0.2682926829268293, "arts": 0.53125, "astronomy": 0.30303030303030304, "business_ethics": 0.5358851674641149, "chinese_civil_service_exam": 0.33125, "chinese_driving_rule": 0.5801526717557252, "chinese_food_culture": 0.27941176470588236, "chinese_foreign_policy": 0.4392523364485981, "chinese_history": 0.4674922600619195, "chinese_literature": 0.28431372549019607, "chinese_teacher_qualification": 0.5139664804469274, "clinical_knowledge": 0.3333333333333333, "college_actuarial_science": 0.22641509433962265, "college_education": 0.5887850467289719, "college_engineering_hydrology": 0.42452830188679247, "college_law": 0.35185185185185186, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.3490566037735849, "college_medicine": 0.38095238095238093, "computer_science": 0.5686274509803921, "computer_security": 0.5555555555555556, "conceptual_physics": 0.3469387755102041, "construction_project_management": 0.3669064748201439, "economics": 0.5220125786163522, "education": 0.50920245398773, "electrical_engineering": 0.45930232558139533, "elementary_chinese": 0.26587301587301587, "elementary_commonsense": 0.4090909090909091, "elementary_information_and_technology": 0.6932773109243697, "elementary_mathematics": 0.32608695652173914, "ethnology": 0.4222222222222222, "food_science": 0.4125874125874126, "genetics": 0.4034090909090909, "global_facts": 0.47651006711409394, "high_school_biology": 0.3136094674556213, "high_school_chemistry": 0.3181818181818182, "high_school_geography": 0.4067796610169492, "high_school_mathematics": 0.3048780487804878, "high_school_physics": 0.43636363636363634, "high_school_politics": 0.38461538461538464, "human_sexuality": 0.5714285714285714, "international_law": 0.3567567567567568, "journalism": 0.4476744186046512, "jurisprudence": 0.45498783454987834, "legal_and_moral_basis": 0.705607476635514, "logical": 0.42276422764227645, "machine_learning": 0.45901639344262296, "management": 0.5952380952380952, "marketing": 0.6111111111111112, "marxist_theory": 0.5185185185185185, "modern_chinese": 0.3017241379310345, "nutrition": 0.5103448275862069, "philosophy": 0.44761904761904764, "professional_accounting": 0.44, "professional_law": 0.33175355450236965, "professional_medicine": 0.27393617021276595, "professional_psychology": 0.4956896551724138, "public_relations": 0.47126436781609193, "security_study": 0.5703703703703704, "sociology": 0.5088495575221239, "sports_science": 0.509090909090909, "traditional_chinese_medicine": 0.32972972972972975, "virology": 0.4970414201183432, "world_history": 0.4782608695652174, "world_religions": 0.5125 } } }, "zbench": { "prompt_1": { "accuracy": 0.3333333333333333 }, "prompt_2": { "accuracy": 0.3333333333333333 }, "prompt_3": { "accuracy": 0.3333333333333333 }, "prompt_4": { "accuracy": 0.3939393939393939 }, "prompt_5": { "accuracy": 0.36363636363636365 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6227272727272727 }, "prompt_2": { "accuracy": 0.6431818181818182 }, "prompt_3": { "accuracy": 0.6090909090909091 }, "prompt_4": { "accuracy": 0.5863636363636363 }, "prompt_5": { "accuracy": 0.6272727272727273 } }, "ocnli": { "prompt_1": { "accuracy": 0.5227118644067796 }, "prompt_2": { "accuracy": 0.49254237288135594 }, "prompt_3": { "accuracy": 0.5033898305084745 }, "prompt_4": { "accuracy": 0.5271186440677966 }, "prompt_5": { "accuracy": 0.5393220338983051 } }, "c3": { "prompt_1": { "accuracy": 0.8163799551234107 }, "prompt_2": { "accuracy": 0.8156320119670905 }, "prompt_3": { "accuracy": 0.8148840688107704 }, "prompt_4": { "accuracy": 0.8156320119670905 }, "prompt_5": { "accuracy": 0.8178758414360509 } }, "dream": { "prompt_1": { "accuracy": 0.8721215090641842 }, "prompt_2": { "accuracy": 0.8652621264086232 }, "prompt_3": { "accuracy": 0.8931896129348359 }, "prompt_4": { "accuracy": 0.8775110240078393 }, "prompt_5": { "accuracy": 0.9039686428221461 } }, "samsum": { "prompt_1": { "rouge1": 0.3379569725563289, "rouge2": 0.1297505746252268, "rougeL": 0.2583327984045818, "avg_rouge": 0.2420134485287125 }, "prompt_2": { "rouge1": 0.36693774963873615, "rouge2": 0.13995152326152985, "rougeL": 0.28220133592060764, "avg_rouge": 0.26303020294029117 }, "prompt_3": { "rouge1": 0.35187138311903504, "rouge2": 0.13105884717466826, "rougeL": 0.2673882223081265, "avg_rouge": 0.2501061508672766 }, "prompt_4": { "rouge1": 0.35465837446263077, "rouge2": 0.13333498040065456, "rougeL": 0.27015665755068613, "avg_rouge": 0.25271667080465715 }, "prompt_5": { "rouge1": 0.3742821190677485, "rouge2": 0.14040447358984648, "rougeL": 0.28825299811660415, "avg_rouge": 0.2676465302580664 } }, "dialogsum": { "prompt_1": { "rouge1": 0.23089767016259732, "rouge2": 0.06172943469639488, "rougeL": 0.17016908958572435, "avg_rouge": 0.15426539814823884 }, "prompt_2": { "rouge1": 0.22861164883981736, "rouge2": 0.059782881376602634, "rougeL": 0.16816208890349965, "avg_rouge": 0.15218553970663987 }, "prompt_3": { "rouge1": 0.2294992806454931, "rouge2": 0.060170947657632076, "rougeL": 0.16849272526047757, "avg_rouge": 0.1527209845212009 }, "prompt_4": { "rouge1": 0.23145317379810715, "rouge2": 0.06210396398371093, "rougeL": 0.17045338878061678, "avg_rouge": 0.1546701755208116 }, "prompt_5": { "rouge1": 0.22469808863669913, "rouge2": 0.056208955341898646, "rougeL": 0.1678153274684139, "avg_rouge": 0.14957412381567056 } }, "sst2": { "prompt_1": { "accuracy": 0.9403669724770642 }, "prompt_2": { "accuracy": 0.9380733944954128 }, "prompt_3": { "accuracy": 0.9461009174311926 }, "prompt_4": { "accuracy": 0.9346330275229358 }, "prompt_5": { "accuracy": 0.908256880733945 } }, "cola": { "prompt_1": { "accuracy": 0.7689357622243528 }, "prompt_2": { "accuracy": 0.7545541706615532 }, "prompt_3": { "accuracy": 0.7900287631831256 }, "prompt_4": { "accuracy": 0.8015340364333653 }, "prompt_5": { "accuracy": 0.7727708533077661 } }, "qqp": { "prompt_1": { "accuracy": 0.7925 }, "prompt_2": { "accuracy": 0.792 }, "prompt_3": { "accuracy": 0.7845 }, "prompt_4": { "accuracy": 0.779 }, "prompt_5": { "accuracy": 0.78 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.7 } }, "qnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.8 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "gpt_35_turbo_1106": { "model_size": "NA", "model_link": "https://openai.com/blog/chatgpt", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.915126050420168, "language_acc": { "Spanish": 0.9201680672268907, "English": 0.9436974789915966, "Chinese": 0.9042016806722689, "Vietnamese": 0.892436974789916 }, "consistency_score_2": 0.900280112044818, "consistency_score_3": 0.8569327731092438, "consistency_score_4": 0.826890756302521, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9285714285714286, "Spanish,Chinese": 0.8949579831932774, "Spanish,Vietnamese": 0.892436974789916, "English,Chinese": 0.9050420168067227, "English,Vietnamese": 0.9033613445378151, "Chinese,Vietnamese": 0.8773109243697479 }, "3_combine": { "Spanish,English,Chinese": 0.8697478991596639, "Spanish,English,Vietnamese": 0.8672268907563025, "Spanish,Chinese,Vietnamese": 0.8411764705882353, "English,Chinese,Vietnamese": 0.8495798319327731 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.826890756302521 } }, "AC3_2": 0.9076423779936448, "AC3_3": 0.8850738967286825, "AC3_4": 0.8687737902187682 }, "prompt_2": { "overall_acc": 0.9155462184873949, "language_acc": { "Spanish": 0.9193277310924369, "English": 0.9436974789915966, "Chinese": 0.9058823529411765, "Vietnamese": 0.8932773109243698 }, "consistency_score_2": 0.9009803921568628, "consistency_score_3": 0.8571428571428572, "consistency_score_4": 0.826890756302521, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.926890756302521, "Spanish,Chinese": 0.8966386554621849, "Spanish,Vietnamese": 0.892436974789916, "English,Chinese": 0.9067226890756303, "English,Vietnamese": 0.9033613445378151, "Chinese,Vietnamese": 0.8798319327731092 }, "3_combine": { "Spanish,English,Chinese": 0.8697478991596639, "Spanish,English,Vietnamese": 0.865546218487395, "Spanish,Chinese,Vietnamese": 0.8420168067226891, "English,Chinese,Vietnamese": 0.8512605042016806 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.826890756302521 } }, "AC3_2": 0.9082049072020327, "AC3_3": 0.8853824534766179, "AC3_4": 0.8689630855460749 }, "prompt_3": { "overall_acc": 0.9065126050420168, "language_acc": { "Spanish": 0.9159663865546218, "English": 0.9277310924369748, "Chinese": 0.9008403361344538, "Vietnamese": 0.8815126050420168 }, "consistency_score_2": 0.8900560224089636, "consistency_score_3": 0.8407563025210084, "consistency_score_4": 0.8050420168067227, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9184873949579831, "Spanish,Chinese": 0.892436974789916, "Spanish,Vietnamese": 0.873109243697479, "English,Chinese": 0.8974789915966387, "English,Vietnamese": 0.8890756302521008, "Chinese,Vietnamese": 0.8697478991596639 }, "3_combine": { "Spanish,English,Chinese": 0.8571428571428571, "Spanish,English,Vietnamese": 0.846218487394958, "Spanish,Chinese,Vietnamese": 0.8252100840336134, "English,Chinese,Vietnamese": 0.8344537815126051 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8050420168067227 } }, "AC3_2": 0.8982089424628893, "AC3_3": 0.8723971252063294, "AC3_4": 0.8527694371714862 }, "prompt_4": { "overall_acc": 0.9142857142857143, "language_acc": { "Spanish": 0.9319327731092437, "English": 0.938655462184874, "Chinese": 0.9016806722689076, "Vietnamese": 0.8848739495798319 }, "consistency_score_2": 0.8978991596638656, "consistency_score_3": 0.8525210084033614, "consistency_score_4": 0.8184873949579832, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.938655462184874, "Spanish,Chinese": 0.8983193277310925, "Spanish,Vietnamese": 0.8949579831932774, "English,Chinese": 0.8983193277310925, "English,Vietnamese": 0.8865546218487395, "Chinese,Vietnamese": 0.8705882352941177 }, "3_combine": { "Spanish,English,Chinese": 0.8705882352941177, "Spanish,English,Vietnamese": 0.8630252100840337, "Spanish,Chinese,Vietnamese": 0.8411764705882353, "English,Chinese,Vietnamese": 0.8352941176470589 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8184873949579832 } }, "AC3_2": 0.9060183497893602, "AC3_3": 0.8823237641763225, "AC3_4": 0.863738395405328 }, "prompt_5": { "overall_acc": 0.9058823529411765, "language_acc": { "Spanish": 0.9168067226890756, "English": 0.926890756302521, "Chinese": 0.9025210084033614, "Vietnamese": 0.8773109243697479 }, "consistency_score_2": 0.8901960784313726, "consistency_score_3": 0.8418067226890756, "consistency_score_4": 0.8067226890756303, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9193277310924369, "Spanish,Chinese": 0.8957983193277311, "Spanish,Vietnamese": 0.8697478991596639, "English,Chinese": 0.9016806722689076, "English,Vietnamese": 0.8899159663865546, "Chinese,Vietnamese": 0.8647058823529412 }, "3_combine": { "Spanish,English,Chinese": 0.8630252100840337, "Spanish,English,Vietnamese": 0.8453781512605042, "Spanish,Chinese,Vietnamese": 0.8235294117647058, "English,Chinese,Vietnamese": 0.8352941176470589 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8067226890756303 } }, "AC3_2": 0.8979707166209516, "AC3_3": 0.8726699334120924, "AC3_4": 0.8534318535552013 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.661904761904762, "language_acc": { "Vietnamese": 0.6133333333333333, "Malay": 0.6133333333333333, "Filipino": 0.6266666666666667, "Indonesian": 0.66, "Chinese": 0.6533333333333333, "Spanish": 0.72, "English": 0.7466666666666667 }, "consistency_score_2": 0.6663492063492062, "consistency_score_3": 0.5316190476190475, "consistency_score_4": 0.449904761904762, "consistency_score_5": 0.3923809523809524, "consistency_score_6": 0.3485714285714286, "consistency_score_7": 0.31333333333333335, "detailed_consistency_score": { "2_combine": { "Vietnamese,Malay": 0.5666666666666667, "Vietnamese,Filipino": 0.6133333333333333, "Vietnamese,Indonesian": 0.6466666666666666, "Vietnamese,Chinese": 0.5933333333333334, "Vietnamese,Spanish": 0.6666666666666666, "Vietnamese,English": 0.64, "Malay,Filipino": 0.62, "Malay,Indonesian": 0.6866666666666666, "Malay,Chinese": 0.64, "Malay,Spanish": 0.6933333333333334, "Malay,English": 0.6733333333333333, "Filipino,Indonesian": 0.6733333333333333, "Filipino,Chinese": 0.6133333333333333, "Filipino,Spanish": 0.72, "Filipino,English": 0.6666666666666666, "Indonesian,Chinese": 0.6666666666666666, "Indonesian,Spanish": 0.76, "Indonesian,English": 0.7333333333333333, "Chinese,Spanish": 0.6666666666666666, "Chinese,English": 0.6533333333333333, "Spanish,English": 0.8 }, "3_combine": { "Vietnamese,Malay,Filipino": 0.4533333333333333, "Vietnamese,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Malay,Chinese": 0.4533333333333333, "Vietnamese,Malay,Spanish": 0.49333333333333335, "Vietnamese,Malay,English": 0.4866666666666667, "Vietnamese,Filipino,Indonesian": 0.5133333333333333, "Vietnamese,Filipino,Chinese": 0.4533333333333333, "Vietnamese,Filipino,Spanish": 0.54, "Vietnamese,Filipino,English": 0.5133333333333333, "Vietnamese,Indonesian,Chinese": 0.5, "Vietnamese,Indonesian,Spanish": 0.5733333333333334, "Vietnamese,Indonesian,English": 0.56, "Vietnamese,Chinese,Spanish": 0.5, "Vietnamese,Chinese,English": 0.4866666666666667, "Vietnamese,Spanish,English": 0.58, "Malay,Filipino,Indonesian": 0.5266666666666666, "Malay,Filipino,Chinese": 0.48, "Malay,Filipino,Spanish": 0.54, "Malay,Filipino,English": 0.52, "Malay,Indonesian,Chinese": 0.5266666666666666, "Malay,Indonesian,Spanish": 0.5933333333333334, "Malay,Indonesian,English": 0.5733333333333334, "Malay,Chinese,Spanish": 0.5133333333333333, "Malay,Chinese,English": 0.5133333333333333, "Malay,Spanish,English": 0.6066666666666667, "Filipino,Indonesian,Chinese": 0.49333333333333335, "Filipino,Indonesian,Spanish": 0.5866666666666667, "Filipino,Indonesian,English": 0.5533333333333333, "Filipino,Chinese,Spanish": 0.5133333333333333, "Filipino,Chinese,English": 0.5, "Filipino,Spanish,English": 0.6066666666666667, "Indonesian,Chinese,Spanish": 0.5666666666666667, "Indonesian,Chinese,English": 0.5466666666666666, "Indonesian,Spanish,English": 0.6733333333333333, "Chinese,Spanish,English": 0.58 }, "4_combine": { "Vietnamese,Malay,Filipino,Indonesian": 0.41333333333333333, "Vietnamese,Malay,Filipino,Chinese": 0.38, "Vietnamese,Malay,Filipino,Spanish": 0.4266666666666667, "Vietnamese,Malay,Filipino,English": 0.4066666666666667, "Vietnamese,Malay,Indonesian,Chinese": 0.4066666666666667, "Vietnamese,Malay,Indonesian,Spanish": 0.4533333333333333, "Vietnamese,Malay,Indonesian,English": 0.4533333333333333, "Vietnamese,Malay,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Malay,Chinese,English": 0.4, "Vietnamese,Malay,Spanish,English": 0.46, "Vietnamese,Filipino,Indonesian,Chinese": 0.4, "Vietnamese,Filipino,Indonesian,Spanish": 0.48, "Vietnamese,Filipino,Indonesian,English": 0.46, "Vietnamese,Filipino,Chinese,Spanish": 0.41333333333333333, "Vietnamese,Filipino,Chinese,English": 0.4, "Vietnamese,Filipino,Spanish,English": 0.48, "Vietnamese,Indonesian,Chinese,Spanish": 0.46, "Vietnamese,Indonesian,Chinese,English": 0.4533333333333333, "Vietnamese,Indonesian,Spanish,English": 0.5266666666666666, "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, "Malay,Filipino,Indonesian,Chinese": 0.41333333333333333, "Malay,Filipino,Indonesian,Spanish": 0.4866666666666667, "Malay,Filipino,Indonesian,English": 0.4666666666666667, "Malay,Filipino,Chinese,Spanish": 0.43333333333333335, "Malay,Filipino,Chinese,English": 0.42, "Malay,Filipino,Spanish,English": 0.48, "Malay,Indonesian,Chinese,Spanish": 0.46, "Malay,Indonesian,Chinese,English": 0.46, "Malay,Indonesian,Spanish,English": 0.54, "Malay,Chinese,Spanish,English": 0.4666666666666667, "Filipino,Indonesian,Chinese,Spanish": 0.44666666666666666, "Filipino,Indonesian,Chinese,English": 0.43333333333333335, "Filipino,Indonesian,Spanish,English": 0.5266666666666666, "Filipino,Chinese,Spanish,English": 0.4666666666666667, "Indonesian,Chinese,Spanish,English": 0.5133333333333333 }, "5_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.34, "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4, "Vietnamese,Malay,Filipino,Indonesian,English": 0.38666666666666666, "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.36, "Vietnamese,Malay,Filipino,Chinese,English": 0.34, "Vietnamese,Malay,Filipino,Spanish,English": 0.3933333333333333, "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.38, "Vietnamese,Malay,Indonesian,Chinese,English": 0.38, "Vietnamese,Malay,Indonesian,Spanish,English": 0.4266666666666667, "Vietnamese,Malay,Chinese,Spanish,English": 0.38, "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.38, "Vietnamese,Filipino,Indonesian,Chinese,English": 0.36666666666666664, "Vietnamese,Filipino,Indonesian,Spanish,English": 0.44, "Vietnamese,Filipino,Chinese,Spanish,English": 0.38, "Vietnamese,Indonesian,Chinese,Spanish,English": 0.4266666666666667, "Malay,Filipino,Indonesian,Chinese,Spanish": 0.38666666666666666, "Malay,Filipino,Indonesian,Chinese,English": 0.38, "Malay,Filipino,Indonesian,Spanish,English": 0.44666666666666666, "Malay,Filipino,Chinese,Spanish,English": 0.4, "Malay,Indonesian,Chinese,Spanish,English": 0.43333333333333335, "Filipino,Indonesian,Chinese,Spanish,English": 0.41333333333333333 }, "6_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.3333333333333333, "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.32, "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.37333333333333335, "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.3333333333333333, "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.36, "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.35333333333333333, "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664 }, "7_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.31333333333333335 } }, "AC3_2": 0.6641195483429715, "AC3_3": 0.5896508746637111, "AC3_4": 0.5356926666021684, "AC3_5": 0.49269153004317184, "AC3_6": 0.45665813917656617, "AC3_7": 0.4253255207897209 }, "prompt_2": { "overall_acc": 0.6885714285714286, "language_acc": { "Vietnamese": 0.6333333333333333, "Malay": 0.6533333333333333, "Filipino": 0.6666666666666666, "Indonesian": 0.6933333333333334, "Chinese": 0.6533333333333333, "Spanish": 0.7333333333333333, "English": 0.7866666666666666 }, "consistency_score_2": 0.6926984126984127, "consistency_score_3": 0.5647619047619048, "consistency_score_4": 0.4864761904761906, "consistency_score_5": 0.43142857142857144, "consistency_score_6": 0.3904761904761905, "consistency_score_7": 0.36, "detailed_consistency_score": { "2_combine": { "Vietnamese,Malay": 0.62, "Vietnamese,Filipino": 0.66, "Vietnamese,Indonesian": 0.7066666666666667, "Vietnamese,Chinese": 0.6533333333333333, "Vietnamese,Spanish": 0.6666666666666666, "Vietnamese,English": 0.68, "Malay,Filipino": 0.6466666666666666, "Malay,Indonesian": 0.6866666666666666, "Malay,Chinese": 0.66, "Malay,Spanish": 0.68, "Malay,English": 0.6733333333333333, "Filipino,Indonesian": 0.72, "Filipino,Chinese": 0.6066666666666667, "Filipino,Spanish": 0.7066666666666667, "Filipino,English": 0.7333333333333333, "Indonesian,Chinese": 0.6933333333333334, "Indonesian,Spanish": 0.7666666666666667, "Indonesian,English": 0.7933333333333333, "Chinese,Spanish": 0.6533333333333333, "Chinese,English": 0.7, "Spanish,English": 0.84 }, "3_combine": { "Vietnamese,Malay,Filipino": 0.49333333333333335, "Vietnamese,Malay,Indonesian": 0.5333333333333333, "Vietnamese,Malay,Chinese": 0.5, "Vietnamese,Malay,Spanish": 0.5066666666666667, "Vietnamese,Malay,English": 0.52, "Vietnamese,Filipino,Indonesian": 0.5666666666666667, "Vietnamese,Filipino,Chinese": 0.4866666666666667, "Vietnamese,Filipino,Spanish": 0.54, "Vietnamese,Filipino,English": 0.5666666666666667, "Vietnamese,Indonesian,Chinese": 0.5466666666666666, "Vietnamese,Indonesian,Spanish": 0.58, "Vietnamese,Indonesian,English": 0.6133333333333333, "Vietnamese,Chinese,Spanish": 0.5066666666666667, "Vietnamese,Chinese,English": 0.54, "Vietnamese,Spanish,English": 0.6066666666666667, "Malay,Filipino,Indonesian": 0.56, "Malay,Filipino,Chinese": 0.49333333333333335, "Malay,Filipino,Spanish": 0.5533333333333333, "Malay,Filipino,English": 0.56, "Malay,Indonesian,Chinese": 0.54, "Malay,Indonesian,Spanish": 0.5933333333333334, "Malay,Indonesian,English": 0.6066666666666667, "Malay,Chinese,Spanish": 0.5333333333333333, "Malay,Chinese,English": 0.5533333333333333, "Malay,Spanish,English": 0.6133333333333333, "Filipino,Indonesian,Chinese": 0.5333333333333333, "Filipino,Indonesian,Spanish": 0.6333333333333333, "Filipino,Indonesian,English": 0.6533333333333333, "Filipino,Chinese,Spanish": 0.52, "Filipino,Chinese,English": 0.5533333333333333, "Filipino,Spanish,English": 0.66, "Indonesian,Chinese,Spanish": 0.5733333333333334, "Indonesian,Chinese,English": 0.6133333333333333, "Indonesian,Spanish,English": 0.7066666666666667, "Chinese,Spanish,English": 0.6066666666666667 }, "4_combine": { "Vietnamese,Malay,Filipino,Indonesian": 0.46, "Vietnamese,Malay,Filipino,Chinese": 0.4066666666666667, "Vietnamese,Malay,Filipino,Spanish": 0.44, "Vietnamese,Malay,Filipino,English": 0.46, "Vietnamese,Malay,Indonesian,Chinese": 0.44, "Vietnamese,Malay,Indonesian,Spanish": 0.48, "Vietnamese,Malay,Indonesian,English": 0.4866666666666667, "Vietnamese,Malay,Chinese,Spanish": 0.4266666666666667, "Vietnamese,Malay,Chinese,English": 0.44666666666666666, "Vietnamese,Malay,Spanish,English": 0.47333333333333333, "Vietnamese,Filipino,Indonesian,Chinese": 0.44, "Vietnamese,Filipino,Indonesian,Spanish": 0.5066666666666667, "Vietnamese,Filipino,Indonesian,English": 0.52, "Vietnamese,Filipino,Chinese,Spanish": 0.4266666666666667, "Vietnamese,Filipino,Chinese,English": 0.46, "Vietnamese,Filipino,Spanish,English": 0.52, "Vietnamese,Indonesian,Chinese,Spanish": 0.4666666666666667, "Vietnamese,Indonesian,Chinese,English": 0.5, "Vietnamese,Indonesian,Spanish,English": 0.5533333333333333, "Vietnamese,Chinese,Spanish,English": 0.4866666666666667, "Malay,Filipino,Indonesian,Chinese": 0.44666666666666666, "Malay,Filipino,Indonesian,Spanish": 0.5133333333333333, "Malay,Filipino,Indonesian,English": 0.5266666666666666, "Malay,Filipino,Chinese,Spanish": 0.44666666666666666, "Malay,Filipino,Chinese,English": 0.47333333333333333, "Malay,Filipino,Spanish,English": 0.5133333333333333, "Malay,Indonesian,Chinese,Spanish": 0.4866666666666667, "Malay,Indonesian,Chinese,English": 0.5066666666666667, "Malay,Indonesian,Spanish,English": 0.56, "Malay,Chinese,Spanish,English": 0.5133333333333333, "Filipino,Indonesian,Chinese,Spanish": 0.47333333333333333, "Filipino,Indonesian,Chinese,English": 0.5, "Filipino,Indonesian,Spanish,English": 0.6066666666666667, "Filipino,Chinese,Spanish,English": 0.5066666666666667, "Indonesian,Chinese,Spanish,English": 0.5533333333333333 }, "5_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.38, "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4266666666666667, "Vietnamese,Malay,Filipino,Indonesian,English": 0.43333333333333335, "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.37333333333333335, "Vietnamese,Malay,Filipino,Chinese,English": 0.4, "Vietnamese,Malay,Filipino,Spanish,English": 0.42, "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Malay,Indonesian,Chinese,English": 0.42, "Vietnamese,Malay,Indonesian,Spanish,English": 0.4533333333333333, "Vietnamese,Malay,Chinese,Spanish,English": 0.41333333333333333, "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.4, "Vietnamese,Filipino,Indonesian,Chinese,English": 0.42, "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4866666666666667, "Vietnamese,Filipino,Chinese,Spanish,English": 0.4266666666666667, "Vietnamese,Indonesian,Chinese,Spanish,English": 0.46, "Malay,Filipino,Indonesian,Chinese,Spanish": 0.42, "Malay,Filipino,Indonesian,Chinese,English": 0.44, "Malay,Filipino,Indonesian,Spanish,English": 0.49333333333333335, "Malay,Filipino,Chinese,Spanish,English": 0.44, "Malay,Indonesian,Chinese,Spanish,English": 0.48, "Filipino,Indonesian,Chinese,Spanish,English": 0.4666666666666667 }, "6_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.36, "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.37333333333333335, "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.4066666666666667, "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.37333333333333335, "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.4, "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4, "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.42 }, "7_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36 } }, "AC3_2": 0.6906287552443499, "AC3_3": 0.6205514545748909, "AC3_4": 0.570144732832474, "AC3_5": 0.5304810495153178, "AC3_6": 0.49834825364187546, "AC3_7": 0.4728065394644463 }, "prompt_3": { "overall_acc": 0.7009523809523809, "language_acc": { "Vietnamese": 0.6266666666666667, "Malay": 0.68, "Filipino": 0.6933333333333334, "Indonesian": 0.6933333333333334, "Chinese": 0.6533333333333333, "Spanish": 0.7333333333333333, "English": 0.8266666666666667 }, "consistency_score_2": 0.6974603174603174, "consistency_score_3": 0.5767619047619048, "consistency_score_4": 0.5036190476190475, "consistency_score_5": 0.45269841269841266, "consistency_score_6": 0.41523809523809524, "consistency_score_7": 0.38666666666666666, "detailed_consistency_score": { "2_combine": { "Vietnamese,Malay": 0.6466666666666666, "Vietnamese,Filipino": 0.68, "Vietnamese,Indonesian": 0.64, "Vietnamese,Chinese": 0.62, "Vietnamese,Spanish": 0.6533333333333333, "Vietnamese,English": 0.64, "Malay,Filipino": 0.7, "Malay,Indonesian": 0.7466666666666667, "Malay,Chinese": 0.6466666666666666, "Malay,Spanish": 0.7666666666666667, "Malay,English": 0.74, "Filipino,Indonesian": 0.7, "Filipino,Chinese": 0.6466666666666666, "Filipino,Spanish": 0.7266666666666667, "Filipino,English": 0.7466666666666667, "Indonesian,Chinese": 0.6466666666666666, "Indonesian,Spanish": 0.74, "Indonesian,English": 0.76, "Chinese,Spanish": 0.68, "Chinese,English": 0.6866666666666666, "Spanish,English": 0.8333333333333334 }, "3_combine": { "Vietnamese,Malay,Filipino": 0.5466666666666666, "Vietnamese,Malay,Indonesian": 0.5533333333333333, "Vietnamese,Malay,Chinese": 0.5133333333333333, "Vietnamese,Malay,Spanish": 0.56, "Vietnamese,Malay,English": 0.5466666666666666, "Vietnamese,Filipino,Indonesian": 0.5533333333333333, "Vietnamese,Filipino,Chinese": 0.5133333333333333, "Vietnamese,Filipino,Spanish": 0.5666666666666667, "Vietnamese,Filipino,English": 0.5666666666666667, "Vietnamese,Indonesian,Chinese": 0.49333333333333335, "Vietnamese,Indonesian,Spanish": 0.5466666666666666, "Vietnamese,Indonesian,English": 0.56, "Vietnamese,Chinese,Spanish": 0.52, "Vietnamese,Chinese,English": 0.52, "Vietnamese,Spanish,English": 0.5933333333333334, "Malay,Filipino,Indonesian": 0.6, "Malay,Filipino,Chinese": 0.5266666666666666, "Malay,Filipino,Spanish": 0.62, "Malay,Filipino,English": 0.62, "Malay,Indonesian,Chinese": 0.5466666666666666, "Malay,Indonesian,Spanish": 0.64, "Malay,Indonesian,English": 0.6466666666666666, "Malay,Chinese,Spanish": 0.5733333333333334, "Malay,Chinese,English": 0.5666666666666667, "Malay,Spanish,English": 0.6866666666666666, "Filipino,Indonesian,Chinese": 0.5266666666666666, "Filipino,Indonesian,Spanish": 0.62, "Filipino,Indonesian,English": 0.6333333333333333, "Filipino,Chinese,Spanish": 0.5666666666666667, "Filipino,Chinese,English": 0.56, "Filipino,Spanish,English": 0.6666666666666666, "Indonesian,Chinese,Spanish": 0.56, "Indonesian,Chinese,English": 0.5666666666666667, "Indonesian,Spanish,English": 0.6866666666666666, "Chinese,Spanish,English": 0.62 }, "4_combine": { "Vietnamese,Malay,Filipino,Indonesian": 0.49333333333333335, "Vietnamese,Malay,Filipino,Chinese": 0.44666666666666666, "Vietnamese,Malay,Filipino,Spanish": 0.5066666666666667, "Vietnamese,Malay,Filipino,English": 0.5, "Vietnamese,Malay,Indonesian,Chinese": 0.44, "Vietnamese,Malay,Indonesian,Spanish": 0.5, "Vietnamese,Malay,Indonesian,English": 0.49333333333333335, "Vietnamese,Malay,Chinese,Spanish": 0.47333333333333333, "Vietnamese,Malay,Chinese,English": 0.4666666666666667, "Vietnamese,Malay,Spanish,English": 0.5266666666666666, "Vietnamese,Filipino,Indonesian,Chinese": 0.44666666666666666, "Vietnamese,Filipino,Indonesian,Spanish": 0.5, "Vietnamese,Filipino,Indonesian,English": 0.5133333333333333, "Vietnamese,Filipino,Chinese,Spanish": 0.47333333333333333, "Vietnamese,Filipino,Chinese,English": 0.47333333333333333, "Vietnamese,Filipino,Spanish,English": 0.54, "Vietnamese,Indonesian,Chinese,Spanish": 0.4533333333333333, "Vietnamese,Indonesian,Chinese,English": 0.46, "Vietnamese,Indonesian,Spanish,English": 0.5266666666666666, "Vietnamese,Chinese,Spanish,English": 0.49333333333333335, "Malay,Filipino,Indonesian,Chinese": 0.4666666666666667, "Malay,Filipino,Indonesian,Spanish": 0.54, "Malay,Filipino,Indonesian,English": 0.5533333333333333, "Malay,Filipino,Chinese,Spanish": 0.5, "Malay,Filipino,Chinese,English": 0.4866666666666667, "Malay,Filipino,Spanish,English": 0.58, "Malay,Indonesian,Chinese,Spanish": 0.5066666666666667, "Malay,Indonesian,Chinese,English": 0.5066666666666667, "Malay,Indonesian,Spanish,English": 0.6066666666666667, "Malay,Chinese,Spanish,English": 0.54, "Filipino,Indonesian,Chinese,Spanish": 0.4866666666666667, "Filipino,Indonesian,Chinese,English": 0.4866666666666667, "Filipino,Indonesian,Spanish,English": 0.5866666666666667, "Filipino,Chinese,Spanish,English": 0.5266666666666666, "Indonesian,Chinese,Spanish,English": 0.5266666666666666 }, "5_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.4066666666666667, "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.46, "Vietnamese,Malay,Filipino,Indonesian,English": 0.46, "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.43333333333333335, "Vietnamese,Malay,Filipino,Chinese,English": 0.4266666666666667, "Vietnamese,Malay,Filipino,Spanish,English": 0.4866666666666667, "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4266666666666667, "Vietnamese,Malay,Indonesian,Chinese,English": 0.42, "Vietnamese,Malay,Indonesian,Spanish,English": 0.48, "Vietnamese,Malay,Chinese,Spanish,English": 0.4533333333333333, "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.42, "Vietnamese,Filipino,Indonesian,Chinese,English": 0.4266666666666667, "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4866666666666667, "Vietnamese,Filipino,Chinese,Spanish,English": 0.4533333333333333, "Vietnamese,Indonesian,Chinese,Spanish,English": 0.44, "Malay,Filipino,Indonesian,Chinese,Spanish": 0.44666666666666666, "Malay,Filipino,Indonesian,Chinese,English": 0.44, "Malay,Filipino,Indonesian,Spanish,English": 0.52, "Malay,Filipino,Chinese,Spanish,English": 0.47333333333333333, "Malay,Indonesian,Chinese,Spanish,English": 0.4866666666666667, "Filipino,Indonesian,Chinese,Spanish,English": 0.46 }, "6_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.4, "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.3933333333333333, "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.44666666666666666, "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.42, "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.41333333333333333, "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4066666666666667, "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.4266666666666667 }, "7_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.38666666666666666 } }, "AC3_2": 0.6992019890316715, "AC3_3": 0.6328216485860745, "AC3_4": 0.5861221047072226, "AC3_5": 0.5501145267757028, "AC3_6": 0.5215277100134094, "AC3_7": 0.4984004669711045 }, "prompt_4": { "overall_acc": 0.6647619047619048, "language_acc": { "Vietnamese": 0.5733333333333334, "Malay": 0.6333333333333333, "Filipino": 0.6266666666666667, "Indonesian": 0.6933333333333334, "Chinese": 0.6266666666666667, "Spanish": 0.72, "English": 0.78 }, "consistency_score_2": 0.6571428571428571, "consistency_score_3": 0.520952380952381, "consistency_score_4": 0.44552380952380943, "consistency_score_5": 0.39587301587301593, "consistency_score_6": 0.35999999999999993, "consistency_score_7": 0.3333333333333333, "detailed_consistency_score": { "2_combine": { "Vietnamese,Malay": 0.58, "Vietnamese,Filipino": 0.6333333333333333, "Vietnamese,Indonesian": 0.6266666666666667, "Vietnamese,Chinese": 0.6533333333333333, "Vietnamese,Spanish": 0.6133333333333333, "Vietnamese,English": 0.6, "Malay,Filipino": 0.62, "Malay,Indonesian": 0.6666666666666666, "Malay,Chinese": 0.5666666666666667, "Malay,Spanish": 0.68, "Malay,English": 0.6266666666666667, "Filipino,Indonesian": 0.7, "Filipino,Chinese": 0.6, "Filipino,Spanish": 0.68, "Filipino,English": 0.68, "Indonesian,Chinese": 0.64, "Indonesian,Spanish": 0.72, "Indonesian,English": 0.7466666666666667, "Chinese,Spanish": 0.6333333333333333, "Chinese,English": 0.7, "Spanish,English": 0.8333333333333334 }, "3_combine": { "Vietnamese,Malay,Filipino": 0.46, "Vietnamese,Malay,Indonesian": 0.47333333333333333, "Vietnamese,Malay,Chinese": 0.4533333333333333, "Vietnamese,Malay,Spanish": 0.49333333333333335, "Vietnamese,Malay,English": 0.4533333333333333, "Vietnamese,Filipino,Indonesian": 0.5133333333333333, "Vietnamese,Filipino,Chinese": 0.48, "Vietnamese,Filipino,Spanish": 0.5, "Vietnamese,Filipino,English": 0.5, "Vietnamese,Indonesian,Chinese": 0.5133333333333333, "Vietnamese,Indonesian,Spanish": 0.5066666666666667, "Vietnamese,Indonesian,English": 0.5333333333333333, "Vietnamese,Chinese,Spanish": 0.4866666666666667, "Vietnamese,Chinese,English": 0.5133333333333333, "Vietnamese,Spanish,English": 0.54, "Malay,Filipino,Indonesian": 0.5333333333333333, "Malay,Filipino,Chinese": 0.44, "Malay,Filipino,Spanish": 0.5133333333333333, "Malay,Filipino,English": 0.5066666666666667, "Malay,Indonesian,Chinese": 0.4666666666666667, "Malay,Indonesian,Spanish": 0.5733333333333334, "Malay,Indonesian,English": 0.56, "Malay,Chinese,Spanish": 0.4866666666666667, "Malay,Chinese,English": 0.4866666666666667, "Malay,Spanish,English": 0.5933333333333334, "Filipino,Indonesian,Chinese": 0.5066666666666667, "Filipino,Indonesian,Spanish": 0.5733333333333334, "Filipino,Indonesian,English": 0.5866666666666667, "Filipino,Chinese,Spanish": 0.5, "Filipino,Chinese,English": 0.52, "Filipino,Spanish,English": 0.6133333333333333, "Indonesian,Chinese,Spanish": 0.5266666666666666, "Indonesian,Chinese,English": 0.56, "Indonesian,Spanish,English": 0.6666666666666666, "Chinese,Spanish,English": 0.6 }, "4_combine": { "Vietnamese,Malay,Filipino,Indonesian": 0.41333333333333333, "Vietnamese,Malay,Filipino,Chinese": 0.38666666666666666, "Vietnamese,Malay,Filipino,Spanish": 0.4266666666666667, "Vietnamese,Malay,Filipino,English": 0.4, "Vietnamese,Malay,Indonesian,Chinese": 0.3933333333333333, "Vietnamese,Malay,Indonesian,Spanish": 0.44, "Vietnamese,Malay,Indonesian,English": 0.4266666666666667, "Vietnamese,Malay,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Malay,Chinese,English": 0.4, "Vietnamese,Malay,Spanish,English": 0.44666666666666666, "Vietnamese,Filipino,Indonesian,Chinese": 0.42, "Vietnamese,Filipino,Indonesian,Spanish": 0.4533333333333333, "Vietnamese,Filipino,Indonesian,English": 0.46, "Vietnamese,Filipino,Chinese,Spanish": 0.41333333333333333, "Vietnamese,Filipino,Chinese,English": 0.43333333333333335, "Vietnamese,Filipino,Spanish,English": 0.4666666666666667, "Vietnamese,Indonesian,Chinese,Spanish": 0.42, "Vietnamese,Indonesian,Chinese,English": 0.46, "Vietnamese,Indonesian,Spanish,English": 0.48, "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, "Malay,Filipino,Indonesian,Chinese": 0.3933333333333333, "Malay,Filipino,Indonesian,Spanish": 0.48, "Malay,Filipino,Indonesian,English": 0.47333333333333333, "Malay,Filipino,Chinese,Spanish": 0.41333333333333333, "Malay,Filipino,Chinese,English": 0.4, "Malay,Filipino,Spanish,English": 0.4866666666666667, "Malay,Indonesian,Chinese,Spanish": 0.44, "Malay,Indonesian,Chinese,English": 0.44, "Malay,Indonesian,Spanish,English": 0.54, "Malay,Chinese,Spanish,English": 0.47333333333333333, "Filipino,Indonesian,Chinese,Spanish": 0.4533333333333333, "Filipino,Indonesian,Chinese,English": 0.47333333333333333, "Filipino,Indonesian,Spanish,English": 0.5333333333333333, "Filipino,Chinese,Spanish,English": 0.47333333333333333, "Indonesian,Chinese,Spanish,English": 0.5066666666666667 }, "5_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.3466666666666667, "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4, "Vietnamese,Malay,Filipino,Indonesian,English": 0.38, "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.36666666666666664, "Vietnamese,Malay,Filipino,Chinese,English": 0.35333333333333333, "Vietnamese,Malay,Filipino,Spanish,English": 0.4, "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.37333333333333335, "Vietnamese,Malay,Indonesian,Chinese,English": 0.37333333333333335, "Vietnamese,Malay,Indonesian,Spanish,English": 0.42, "Vietnamese,Malay,Chinese,Spanish,English": 0.3933333333333333, "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.37333333333333335, "Vietnamese,Filipino,Indonesian,Chinese,English": 0.4, "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4266666666666667, "Vietnamese,Filipino,Chinese,Spanish,English": 0.4, "Vietnamese,Indonesian,Chinese,Spanish,English": 0.41333333333333333, "Malay,Filipino,Indonesian,Chinese,Spanish": 0.38666666666666666, "Malay,Filipino,Indonesian,Chinese,English": 0.38, "Malay,Filipino,Indonesian,Spanish,English": 0.46, "Malay,Filipino,Chinese,Spanish,English": 0.4, "Malay,Indonesian,Chinese,Spanish,English": 0.43333333333333335, "Filipino,Indonesian,Chinese,Spanish,English": 0.43333333333333335 }, "6_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.34, "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.3333333333333333, "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.38, "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.35333333333333333, "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.36666666666666664, "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664, "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.38 }, "7_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.3333333333333333 } }, "AC3_2": 0.6609304239928177, "AC3_3": 0.5841361636531642, "AC3_4": 0.5334973735110832, "AC3_5": 0.4962335199999633, "AC3_6": 0.4670631969804445, "AC3_7": 0.4440203561896099 }, "prompt_5": { "overall_acc": 0.699047619047619, "language_acc": { "Vietnamese": 0.62, "Malay": 0.68, "Filipino": 0.6733333333333333, "Indonesian": 0.7, "Chinese": 0.6933333333333334, "Spanish": 0.7266666666666667, "English": 0.8 }, "consistency_score_2": 0.7066666666666666, "consistency_score_3": 0.5843809523809523, "consistency_score_4": 0.5060952380952382, "consistency_score_5": 0.448888888888889, "consistency_score_6": 0.4038095238095238, "consistency_score_7": 0.36666666666666664, "detailed_consistency_score": { "2_combine": { "Vietnamese,Malay": 0.6533333333333333, "Vietnamese,Filipino": 0.6533333333333333, "Vietnamese,Indonesian": 0.66, "Vietnamese,Chinese": 0.64, "Vietnamese,Spanish": 0.6333333333333333, "Vietnamese,English": 0.6533333333333333, "Malay,Filipino": 0.6866666666666666, "Malay,Indonesian": 0.7466666666666667, "Malay,Chinese": 0.6733333333333333, "Malay,Spanish": 0.72, "Malay,English": 0.74, "Filipino,Indonesian": 0.72, "Filipino,Chinese": 0.68, "Filipino,Spanish": 0.7466666666666667, "Filipino,English": 0.7333333333333333, "Indonesian,Chinese": 0.6933333333333334, "Indonesian,Spanish": 0.7466666666666667, "Indonesian,English": 0.76, "Chinese,Spanish": 0.7066666666666667, "Chinese,English": 0.7666666666666667, "Spanish,English": 0.8266666666666667 }, "3_combine": { "Vietnamese,Malay,Filipino": 0.52, "Vietnamese,Malay,Indonesian": 0.5466666666666666, "Vietnamese,Malay,Chinese": 0.5066666666666667, "Vietnamese,Malay,Spanish": 0.5466666666666666, "Vietnamese,Malay,English": 0.56, "Vietnamese,Filipino,Indonesian": 0.54, "Vietnamese,Filipino,Chinese": 0.5133333333333333, "Vietnamese,Filipino,Spanish": 0.5466666666666666, "Vietnamese,Filipino,English": 0.5533333333333333, "Vietnamese,Indonesian,Chinese": 0.5133333333333333, "Vietnamese,Indonesian,Spanish": 0.5466666666666666, "Vietnamese,Indonesian,English": 0.5666666666666667, "Vietnamese,Chinese,Spanish": 0.5133333333333333, "Vietnamese,Chinese,English": 0.5533333333333333, "Vietnamese,Spanish,English": 0.5866666666666667, "Malay,Filipino,Indonesian": 0.6066666666666667, "Malay,Filipino,Chinese": 0.54, "Malay,Filipino,Spanish": 0.6066666666666667, "Malay,Filipino,English": 0.6066666666666667, "Malay,Indonesian,Chinese": 0.58, "Malay,Indonesian,Spanish": 0.6333333333333333, "Malay,Indonesian,English": 0.6533333333333333, "Malay,Chinese,Spanish": 0.5666666666666667, "Malay,Chinese,English": 0.6066666666666667, "Malay,Spanish,English": 0.66, "Filipino,Indonesian,Chinese": 0.5666666666666667, "Filipino,Indonesian,Spanish": 0.6333333333333333, "Filipino,Indonesian,English": 0.64, "Filipino,Chinese,Spanish": 0.5933333333333334, "Filipino,Chinese,English": 0.6133333333333333, "Filipino,Spanish,English": 0.68, "Indonesian,Chinese,Spanish": 0.5866666666666667, "Indonesian,Chinese,English": 0.6266666666666667, "Indonesian,Spanish,English": 0.68, "Chinese,Spanish,English": 0.66 }, "4_combine": { "Vietnamese,Malay,Filipino,Indonesian": 0.4666666666666667, "Vietnamese,Malay,Filipino,Chinese": 0.43333333333333335, "Vietnamese,Malay,Filipino,Spanish": 0.47333333333333333, "Vietnamese,Malay,Filipino,English": 0.48, "Vietnamese,Malay,Indonesian,Chinese": 0.44666666666666666, "Vietnamese,Malay,Indonesian,Spanish": 0.4866666666666667, "Vietnamese,Malay,Indonesian,English": 0.5, "Vietnamese,Malay,Chinese,Spanish": 0.4533333333333333, "Vietnamese,Malay,Chinese,English": 0.48, "Vietnamese,Malay,Spanish,English": 0.5133333333333333, "Vietnamese,Filipino,Indonesian,Chinese": 0.44, "Vietnamese,Filipino,Indonesian,Spanish": 0.4866666666666667, "Vietnamese,Filipino,Indonesian,English": 0.5, "Vietnamese,Filipino,Chinese,Spanish": 0.46, "Vietnamese,Filipino,Chinese,English": 0.4866666666666667, "Vietnamese,Filipino,Spanish,English": 0.52, "Vietnamese,Indonesian,Chinese,Spanish": 0.44666666666666666, "Vietnamese,Indonesian,Chinese,English": 0.4866666666666667, "Vietnamese,Indonesian,Spanish,English": 0.52, "Vietnamese,Chinese,Spanish,English": 0.5, "Malay,Filipino,Indonesian,Chinese": 0.4866666666666667, "Malay,Filipino,Indonesian,Spanish": 0.5533333333333333, "Malay,Filipino,Indonesian,English": 0.5533333333333333, "Malay,Filipino,Chinese,Spanish": 0.5, "Malay,Filipino,Chinese,English": 0.52, "Malay,Filipino,Spanish,English": 0.5666666666666667, "Malay,Indonesian,Chinese,Spanish": 0.5133333333333333, "Malay,Indonesian,Chinese,English": 0.5466666666666666, "Malay,Indonesian,Spanish,English": 0.5933333333333334, "Malay,Chinese,Spanish,English": 0.54, "Filipino,Indonesian,Chinese,Spanish": 0.5133333333333333, "Filipino,Indonesian,Chinese,English": 0.5333333333333333, "Filipino,Indonesian,Spanish,English": 0.6, "Filipino,Chinese,Spanish,English": 0.56, "Indonesian,Chinese,Spanish,English": 0.5533333333333333 }, "5_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.38666666666666666, "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.43333333333333335, "Vietnamese,Malay,Filipino,Indonesian,English": 0.44, "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Malay,Filipino,Chinese,English": 0.4266666666666667, "Vietnamese,Malay,Filipino,Spanish,English": 0.4533333333333333, "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Malay,Indonesian,Chinese,English": 0.43333333333333335, "Vietnamese,Malay,Indonesian,Spanish,English": 0.4666666666666667, "Vietnamese,Malay,Chinese,Spanish,English": 0.44, "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.4066666666666667, "Vietnamese,Filipino,Indonesian,Chinese,English": 0.43333333333333335, "Vietnamese,Filipino,Indonesian,Spanish,English": 0.47333333333333333, "Vietnamese,Filipino,Chinese,Spanish,English": 0.4533333333333333, "Vietnamese,Indonesian,Chinese,Spanish,English": 0.44, "Malay,Filipino,Indonesian,Chinese,Spanish": 0.46, "Malay,Filipino,Indonesian,Chinese,English": 0.47333333333333333, "Malay,Filipino,Indonesian,Spanish,English": 0.5266666666666666, "Malay,Filipino,Chinese,Spanish,English": 0.48, "Malay,Indonesian,Chinese,Spanish,English": 0.49333333333333335, "Filipino,Indonesian,Chinese,Spanish,English": 0.49333333333333335 }, "6_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.36666666666666664, "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.38666666666666666, "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.42, "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.4, "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.4, "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4066666666666667, "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.44666666666666666 }, "7_combine": { "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664 } }, "AC3_2": 0.7028364949816184, "AC3_3": 0.6365918952422053, "AC3_4": 0.5871248691814143, "AC3_5": 0.5467109144066519, "AC3_6": 0.5119105189107344, "AC3_7": 0.48102472441122346 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.47970779220779225, "language_acc": { "English": 0.5625, "Filipino": 0.3806818181818182, "Vietnamese": 0.4375, "Chinese": 0.5568181818181818, "Indonesian": 0.4375, "Malay": 0.4431818181818182, "Spanish": 0.5397727272727273 }, "consistency_score_2": 0.549512987012987, "consistency_score_3": 0.37905844155844165, "consistency_score_4": 0.29107142857142865, "consistency_score_5": 0.2372835497835498, "consistency_score_6": 0.2012987012987013, "consistency_score_7": 0.17613636363636365, "detailed_consistency_score": { "2_combine": { "English,Filipino": 0.4772727272727273, "English,Vietnamese": 0.5738636363636364, "English,Chinese": 0.5795454545454546, "English,Indonesian": 0.6079545454545454, "English,Malay": 0.6193181818181818, "English,Spanish": 0.7443181818181818, "Filipino,Vietnamese": 0.4602272727272727, "Filipino,Chinese": 0.44886363636363635, "Filipino,Indonesian": 0.44886363636363635, "Filipino,Malay": 0.48863636363636365, "Filipino,Spanish": 0.4602272727272727, "Vietnamese,Chinese": 0.5170454545454546, "Vietnamese,Indonesian": 0.5397727272727273, "Vietnamese,Malay": 0.5170454545454546, "Vietnamese,Spanish": 0.5965909090909091, "Chinese,Indonesian": 0.5511363636363636, "Chinese,Malay": 0.5170454545454546, "Chinese,Spanish": 0.6022727272727273, "Indonesian,Malay": 0.5909090909090909, "Indonesian,Spanish": 0.625, "Malay,Spanish": 0.5738636363636364 }, "3_combine": { "English,Filipino,Vietnamese": 0.3068181818181818, "English,Filipino,Chinese": 0.3125, "English,Filipino,Indonesian": 0.3409090909090909, "English,Filipino,Malay": 0.35795454545454547, "English,Filipino,Spanish": 0.3693181818181818, "English,Vietnamese,Chinese": 0.375, "English,Vietnamese,Indonesian": 0.42045454545454547, "English,Vietnamese,Malay": 0.4090909090909091, "English,Vietnamese,Spanish": 0.48295454545454547, "English,Chinese,Indonesian": 0.42613636363636365, "English,Chinese,Malay": 0.4034090909090909, "English,Chinese,Spanish": 0.48863636363636365, "English,Indonesian,Malay": 0.4659090909090909, "English,Indonesian,Spanish": 0.5170454545454546, "English,Malay,Spanish": 0.5, "Filipino,Vietnamese,Chinese": 0.2840909090909091, "Filipino,Vietnamese,Indonesian": 0.29545454545454547, "Filipino,Vietnamese,Malay": 0.30113636363636365, "Filipino,Vietnamese,Spanish": 0.3125, "Filipino,Chinese,Indonesian": 0.30113636363636365, "Filipino,Chinese,Malay": 0.29545454545454547, "Filipino,Chinese,Spanish": 0.3181818181818182, "Filipino,Indonesian,Malay": 0.32954545454545453, "Filipino,Indonesian,Spanish": 0.3181818181818182, "Filipino,Malay,Spanish": 0.32386363636363635, "Vietnamese,Chinese,Indonesian": 0.375, "Vietnamese,Chinese,Malay": 0.32954545454545453, "Vietnamese,Chinese,Spanish": 0.4090909090909091, "Vietnamese,Indonesian,Malay": 0.3806818181818182, "Vietnamese,Indonesian,Spanish": 0.4431818181818182, "Vietnamese,Malay,Spanish": 0.38636363636363635, "Chinese,Indonesian,Malay": 0.3977272727272727, "Chinese,Indonesian,Spanish": 0.4431818181818182, "Chinese,Malay,Spanish": 0.38636363636363635, "Indonesian,Malay,Spanish": 0.4602272727272727 }, "4_combine": { "English,Filipino,Vietnamese,Chinese": 0.23295454545454544, "English,Filipino,Vietnamese,Indonesian": 0.2556818181818182, "English,Filipino,Vietnamese,Malay": 0.2556818181818182, "English,Filipino,Vietnamese,Spanish": 0.2727272727272727, "English,Filipino,Chinese,Indonesian": 0.25, "English,Filipino,Chinese,Malay": 0.25, "English,Filipino,Chinese,Spanish": 0.2727272727272727, "English,Filipino,Indonesian,Malay": 0.29545454545454547, "English,Filipino,Indonesian,Spanish": 0.2840909090909091, "English,Filipino,Malay,Spanish": 0.29545454545454547, "English,Vietnamese,Chinese,Indonesian": 0.3125, "English,Vietnamese,Chinese,Malay": 0.2840909090909091, "English,Vietnamese,Chinese,Spanish": 0.3409090909090909, "English,Vietnamese,Indonesian,Malay": 0.32954545454545453, "English,Vietnamese,Indonesian,Spanish": 0.375, "English,Vietnamese,Malay,Spanish": 0.3465909090909091, "English,Chinese,Indonesian,Malay": 0.3465909090909091, "English,Chinese,Indonesian,Spanish": 0.38636363636363635, "English,Chinese,Malay,Spanish": 0.35795454545454547, "English,Indonesian,Malay,Spanish": 0.4090909090909091, "Filipino,Vietnamese,Chinese,Indonesian": 0.22727272727272727, "Filipino,Vietnamese,Chinese,Malay": 0.2215909090909091, "Filipino,Vietnamese,Chinese,Spanish": 0.25, "Filipino,Vietnamese,Indonesian,Malay": 0.25, "Filipino,Vietnamese,Indonesian,Spanish": 0.24431818181818182, "Filipino,Vietnamese,Malay,Spanish": 0.23863636363636365, "Filipino,Chinese,Indonesian,Malay": 0.2556818181818182, "Filipino,Chinese,Indonesian,Spanish": 0.24431818181818182, "Filipino,Chinese,Malay,Spanish": 0.24431818181818182, "Filipino,Indonesian,Malay,Spanish": 0.2727272727272727, "Vietnamese,Chinese,Indonesian,Malay": 0.2840909090909091, "Vietnamese,Chinese,Indonesian,Spanish": 0.3409090909090909, "Vietnamese,Chinese,Malay,Spanish": 0.2897727272727273, "Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, "Chinese,Indonesian,Malay,Spanish": 0.3409090909090909 }, "5_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian": 0.20454545454545456, "English,Filipino,Vietnamese,Chinese,Malay": 0.19886363636363635, "English,Filipino,Vietnamese,Chinese,Spanish": 0.2215909090909091, "English,Filipino,Vietnamese,Indonesian,Malay": 0.23295454545454544, "English,Filipino,Vietnamese,Indonesian,Spanish": 0.22727272727272727, "English,Filipino,Vietnamese,Malay,Spanish": 0.2215909090909091, "English,Filipino,Chinese,Indonesian,Malay": 0.22727272727272727, "English,Filipino,Chinese,Indonesian,Spanish": 0.22727272727272727, "English,Filipino,Chinese,Malay,Spanish": 0.22727272727272727, "English,Filipino,Indonesian,Malay,Spanish": 0.2556818181818182, "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, "English,Vietnamese,Chinese,Indonesian,Spanish": 0.2897727272727273, "English,Vietnamese,Chinese,Malay,Spanish": 0.26136363636363635, "English,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, "English,Chinese,Indonesian,Malay,Spanish": 0.3181818181818182, "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.20454545454545456, "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.21022727272727273, "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.20454545454545456, "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.2159090909090909, "Filipino,Chinese,Indonesian,Malay,Spanish": 0.2215909090909091, "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.26136363636363635 }, "6_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.1875, "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.19318181818181818, "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.1875, "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.20454545454545456, "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.21022727272727273, "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.23863636363636365, "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.1875 }, "7_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.17613636363636365 } }, "AC3_2": 0.5122431786957747, "AC3_3": 0.42348495072769815, "AC3_4": 0.3623066854747182, "AC3_5": 0.3175122518547604, "AC3_6": 0.2835936411205318, "AC3_7": 0.2576648289436108 }, "prompt_2": { "overall_acc": 0.4878246753246754, "language_acc": { "English": 0.5909090909090909, "Filipino": 0.4147727272727273, "Vietnamese": 0.4318181818181818, "Chinese": 0.5625, "Indonesian": 0.4772727272727273, "Malay": 0.4318181818181818, "Spanish": 0.5056818181818182 }, "consistency_score_2": 0.5744047619047619, "consistency_score_3": 0.40649350649350646, "consistency_score_4": 0.3180194805194805, "consistency_score_5": 0.26379870129870125, "consistency_score_6": 0.22646103896103895, "consistency_score_7": 0.19886363636363635, "detailed_consistency_score": { "2_combine": { "English,Filipino": 0.5284090909090909, "English,Vietnamese": 0.5454545454545454, "English,Chinese": 0.5795454545454546, "English,Indonesian": 0.6818181818181818, "English,Malay": 0.5795454545454546, "English,Spanish": 0.6875, "Filipino,Vietnamese": 0.5170454545454546, "Filipino,Chinese": 0.5227272727272727, "Filipino,Indonesian": 0.5568181818181818, "Filipino,Malay": 0.5965909090909091, "Filipino,Spanish": 0.5511363636363636, "Vietnamese,Chinese": 0.5170454545454546, "Vietnamese,Indonesian": 0.5625, "Vietnamese,Malay": 0.5284090909090909, "Vietnamese,Spanish": 0.5681818181818182, "Chinese,Indonesian": 0.5795454545454546, "Chinese,Malay": 0.5681818181818182, "Chinese,Spanish": 0.5852272727272727, "Indonesian,Malay": 0.6420454545454546, "Indonesian,Spanish": 0.6136363636363636, "Malay,Spanish": 0.5511363636363636 }, "3_combine": { "English,Filipino,Vietnamese": 0.3522727272727273, "English,Filipino,Chinese": 0.36363636363636365, "English,Filipino,Indonesian": 0.42613636363636365, "English,Filipino,Malay": 0.4034090909090909, "English,Filipino,Spanish": 0.42045454545454547, "English,Vietnamese,Chinese": 0.375, "English,Vietnamese,Indonesian": 0.44886363636363635, "English,Vietnamese,Malay": 0.39204545454545453, "English,Vietnamese,Spanish": 0.42613636363636365, "English,Chinese,Indonesian": 0.44886363636363635, "English,Chinese,Malay": 0.3977272727272727, "English,Chinese,Spanish": 0.4602272727272727, "English,Indonesian,Malay": 0.4772727272727273, "English,Indonesian,Spanish": 0.5227272727272727, "English,Malay,Spanish": 0.4318181818181818, "Filipino,Vietnamese,Chinese": 0.32954545454545453, "Filipino,Vietnamese,Indonesian": 0.3806818181818182, "Filipino,Vietnamese,Malay": 0.375, "Filipino,Vietnamese,Spanish": 0.375, "Filipino,Chinese,Indonesian": 0.3806818181818182, "Filipino,Chinese,Malay": 0.39204545454545453, "Filipino,Chinese,Spanish": 0.3806818181818182, "Filipino,Indonesian,Malay": 0.4431818181818182, "Filipino,Indonesian,Spanish": 0.3977272727272727, "Filipino,Malay,Spanish": 0.4034090909090909, "Vietnamese,Chinese,Indonesian": 0.38636363636363635, "Vietnamese,Chinese,Malay": 0.35795454545454547, "Vietnamese,Chinese,Spanish": 0.38636363636363635, "Vietnamese,Indonesian,Malay": 0.4090909090909091, "Vietnamese,Indonesian,Spanish": 0.42045454545454547, "Vietnamese,Malay,Spanish": 0.39204545454545453, "Chinese,Indonesian,Malay": 0.4147727272727273, "Chinese,Indonesian,Spanish": 0.42613636363636365, "Chinese,Malay,Spanish": 0.39204545454545453, "Indonesian,Malay,Spanish": 0.4375 }, "4_combine": { "English,Filipino,Vietnamese,Chinese": 0.2556818181818182, "English,Filipino,Vietnamese,Indonesian": 0.3181818181818182, "English,Filipino,Vietnamese,Malay": 0.29545454545454547, "English,Filipino,Vietnamese,Spanish": 0.30113636363636365, "English,Filipino,Chinese,Indonesian": 0.3068181818181818, "English,Filipino,Chinese,Malay": 0.29545454545454547, "English,Filipino,Chinese,Spanish": 0.30113636363636365, "English,Filipino,Indonesian,Malay": 0.3465909090909091, "English,Filipino,Indonesian,Spanish": 0.3522727272727273, "English,Filipino,Malay,Spanish": 0.3409090909090909, "English,Vietnamese,Chinese,Indonesian": 0.32386363636363635, "English,Vietnamese,Chinese,Malay": 0.2897727272727273, "English,Vietnamese,Chinese,Spanish": 0.32954545454545453, "English,Vietnamese,Indonesian,Malay": 0.3352272727272727, "English,Vietnamese,Indonesian,Spanish": 0.36363636363636365, "English,Vietnamese,Malay,Spanish": 0.3181818181818182, "English,Chinese,Indonesian,Malay": 0.32954545454545453, "English,Chinese,Indonesian,Spanish": 0.3806818181818182, "English,Chinese,Malay,Spanish": 0.32386363636363635, "English,Indonesian,Malay,Spanish": 0.38636363636363635, "Filipino,Vietnamese,Chinese,Indonesian": 0.26704545454545453, "Filipino,Vietnamese,Chinese,Malay": 0.2784090909090909, "Filipino,Vietnamese,Chinese,Spanish": 0.2840909090909091, "Filipino,Vietnamese,Indonesian,Malay": 0.3125, "Filipino,Vietnamese,Indonesian,Spanish": 0.3181818181818182, "Filipino,Vietnamese,Malay,Spanish": 0.3125, "Filipino,Chinese,Indonesian,Malay": 0.3125, "Filipino,Chinese,Indonesian,Spanish": 0.30113636363636365, "Filipino,Chinese,Malay,Spanish": 0.3125, "Filipino,Indonesian,Malay,Spanish": 0.3465909090909091, "Vietnamese,Chinese,Indonesian,Malay": 0.30113636363636365, "Vietnamese,Chinese,Indonesian,Spanish": 0.32954545454545453, "Vietnamese,Chinese,Malay,Spanish": 0.30113636363636365, "Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, "Chinese,Indonesian,Malay,Spanish": 0.32954545454545453 }, "5_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian": 0.22727272727272727, "English,Filipino,Vietnamese,Chinese,Malay": 0.22727272727272727, "English,Filipino,Vietnamese,Chinese,Spanish": 0.23863636363636365, "English,Filipino,Vietnamese,Indonesian,Malay": 0.26136363636363635, "English,Filipino,Vietnamese,Indonesian,Spanish": 0.2784090909090909, "English,Filipino,Vietnamese,Malay,Spanish": 0.26704545454545453, "English,Filipino,Chinese,Indonesian,Malay": 0.2556818181818182, "English,Filipino,Chinese,Indonesian,Spanish": 0.26704545454545453, "English,Filipino,Chinese,Malay,Spanish": 0.2556818181818182, "English,Filipino,Indonesian,Malay,Spanish": 0.3068181818181818, "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, "English,Vietnamese,Chinese,Indonesian,Spanish": 0.29545454545454547, "English,Vietnamese,Chinese,Malay,Spanish": 0.26136363636363635, "English,Vietnamese,Indonesian,Malay,Spanish": 0.2897727272727273, "English,Chinese,Indonesian,Malay,Spanish": 0.29545454545454547, "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.23295454545454544, "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.25, "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2556818181818182, "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.2784090909090909, "Filipino,Chinese,Indonesian,Malay,Spanish": 0.26704545454545453, "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2727272727272727 }, "6_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.19886363636363635, "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.2215909090909091, "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2159090909090909, "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.24431818181818182, "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.23295454545454544, "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.24431818181818182, "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.22727272727272727 }, "7_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.19886363636363635 } }, "AC3_2": 0.5275862382153784, "AC3_3": 0.44346087742835405, "AC3_4": 0.38503164336923823, "AC3_5": 0.3424255279901211, "AC3_6": 0.30932519181029544, "AC3_7": 0.28254620670718816 }, "prompt_3": -1, "prompt_4": { "overall_acc": 0.5137987012987013, "language_acc": { "English": 0.5681818181818182, "Filipino": 0.4772727272727273, "Vietnamese": 0.4318181818181818, "Chinese": 0.5397727272727273, "Indonesian": 0.5284090909090909, "Malay": 0.5056818181818182, "Spanish": 0.5454545454545454 }, "consistency_score_2": 0.5868506493506492, "consistency_score_3": 0.4237012987012987, "consistency_score_4": 0.34139610389610386, "consistency_score_5": 0.2924783549783549, "consistency_score_6": 0.25892857142857145, "consistency_score_7": 0.23295454545454544, "detailed_consistency_score": { "2_combine": { "English,Filipino": 0.5625, "English,Vietnamese": 0.5852272727272727, "English,Chinese": 0.5681818181818182, "English,Indonesian": 0.6136363636363636, "English,Malay": 0.6022727272727273, "English,Spanish": 0.7443181818181818, "Filipino,Vietnamese": 0.5625, "Filipino,Chinese": 0.5454545454545454, "Filipino,Indonesian": 0.5454545454545454, "Filipino,Malay": 0.5852272727272727, "Filipino,Spanish": 0.5625, "Vietnamese,Chinese": 0.5113636363636364, "Vietnamese,Indonesian": 0.5056818181818182, "Vietnamese,Malay": 0.5454545454545454, "Vietnamese,Spanish": 0.6079545454545454, "Chinese,Indonesian": 0.5454545454545454, "Chinese,Malay": 0.5795454545454546, "Chinese,Spanish": 0.5511363636363636, "Indonesian,Malay": 0.6931818181818182, "Indonesian,Spanish": 0.6306818181818182, "Malay,Spanish": 0.6761363636363636 }, "3_combine": { "English,Filipino,Vietnamese": 0.3977272727272727, "English,Filipino,Chinese": 0.38636363636363635, "English,Filipino,Indonesian": 0.4090909090909091, "English,Filipino,Malay": 0.42613636363636365, "English,Filipino,Spanish": 0.4659090909090909, "English,Vietnamese,Chinese": 0.3693181818181818, "English,Vietnamese,Indonesian": 0.4090909090909091, "English,Vietnamese,Malay": 0.4147727272727273, "English,Vietnamese,Spanish": 0.4943181818181818, "English,Chinese,Indonesian": 0.3977272727272727, "English,Chinese,Malay": 0.42045454545454547, "English,Chinese,Spanish": 0.4772727272727273, "English,Indonesian,Malay": 0.4943181818181818, "English,Indonesian,Spanish": 0.5170454545454546, "English,Malay,Spanish": 0.5397727272727273, "Filipino,Vietnamese,Chinese": 0.3465909090909091, "Filipino,Vietnamese,Indonesian": 0.3806818181818182, "Filipino,Vietnamese,Malay": 0.3977272727272727, "Filipino,Vietnamese,Spanish": 0.4147727272727273, "Filipino,Chinese,Indonesian": 0.375, "Filipino,Chinese,Malay": 0.4090909090909091, "Filipino,Chinese,Spanish": 0.3806818181818182, "Filipino,Indonesian,Malay": 0.4602272727272727, "Filipino,Indonesian,Spanish": 0.4318181818181818, "Filipino,Malay,Spanish": 0.4602272727272727, "Vietnamese,Chinese,Indonesian": 0.32386363636363635, "Vietnamese,Chinese,Malay": 0.36363636363636365, "Vietnamese,Chinese,Spanish": 0.3806818181818182, "Vietnamese,Indonesian,Malay": 0.4090909090909091, "Vietnamese,Indonesian,Spanish": 0.4090909090909091, "Vietnamese,Malay,Spanish": 0.45454545454545453, "Chinese,Indonesian,Malay": 0.4375, "Chinese,Indonesian,Spanish": 0.4034090909090909, "Chinese,Malay,Spanish": 0.4375, "Indonesian,Malay,Spanish": 0.5340909090909091 }, "4_combine": { "English,Filipino,Vietnamese,Chinese": 0.2784090909090909, "English,Filipino,Vietnamese,Indonesian": 0.32954545454545453, "English,Filipino,Vietnamese,Malay": 0.3409090909090909, "English,Filipino,Vietnamese,Spanish": 0.35795454545454547, "English,Filipino,Chinese,Indonesian": 0.3125, "English,Filipino,Chinese,Malay": 0.32954545454545453, "English,Filipino,Chinese,Spanish": 0.3465909090909091, "English,Filipino,Indonesian,Malay": 0.3693181818181818, "English,Filipino,Indonesian,Spanish": 0.3806818181818182, "English,Filipino,Malay,Spanish": 0.3977272727272727, "English,Vietnamese,Chinese,Indonesian": 0.2784090909090909, "English,Vietnamese,Chinese,Malay": 0.29545454545454547, "English,Vietnamese,Chinese,Spanish": 0.32954545454545453, "English,Vietnamese,Indonesian,Malay": 0.3465909090909091, "English,Vietnamese,Indonesian,Spanish": 0.36363636363636365, "English,Vietnamese,Malay,Spanish": 0.39204545454545453, "English,Chinese,Indonesian,Malay": 0.36363636363636365, "English,Chinese,Indonesian,Spanish": 0.36363636363636365, "English,Chinese,Malay,Spanish": 0.39204545454545453, "English,Indonesian,Malay,Spanish": 0.44886363636363635, "Filipino,Vietnamese,Chinese,Indonesian": 0.2784090909090909, "Filipino,Vietnamese,Chinese,Malay": 0.3068181818181818, "Filipino,Vietnamese,Chinese,Spanish": 0.2840909090909091, "Filipino,Vietnamese,Indonesian,Malay": 0.3409090909090909, "Filipino,Vietnamese,Indonesian,Spanish": 0.32954545454545453, "Filipino,Vietnamese,Malay,Spanish": 0.36363636363636365, "Filipino,Chinese,Indonesian,Malay": 0.32954545454545453, "Filipino,Chinese,Indonesian,Spanish": 0.3125, "Filipino,Chinese,Malay,Spanish": 0.3352272727272727, "Filipino,Indonesian,Malay,Spanish": 0.4090909090909091, "Vietnamese,Chinese,Indonesian,Malay": 0.29545454545454547, "Vietnamese,Chinese,Indonesian,Spanish": 0.2897727272727273, "Vietnamese,Chinese,Malay,Spanish": 0.3181818181818182, "Vietnamese,Indonesian,Malay,Spanish": 0.36363636363636365, "Chinese,Indonesian,Malay,Spanish": 0.375 }, "5_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian": 0.25, "English,Filipino,Vietnamese,Chinese,Malay": 0.26136363636363635, "English,Filipino,Vietnamese,Chinese,Spanish": 0.26136363636363635, "English,Filipino,Vietnamese,Indonesian,Malay": 0.30113636363636365, "English,Filipino,Vietnamese,Indonesian,Spanish": 0.3068181818181818, "English,Filipino,Vietnamese,Malay,Spanish": 0.32954545454545453, "English,Filipino,Chinese,Indonesian,Malay": 0.29545454545454547, "English,Filipino,Chinese,Indonesian,Spanish": 0.30113636363636365, "English,Filipino,Chinese,Malay,Spanish": 0.3125, "English,Filipino,Indonesian,Malay,Spanish": 0.35795454545454547, "English,Vietnamese,Chinese,Indonesian,Malay": 0.26136363636363635, "English,Vietnamese,Chinese,Indonesian,Spanish": 0.26136363636363635, "English,Vietnamese,Chinese,Malay,Spanish": 0.2840909090909091, "English,Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, "English,Chinese,Indonesian,Malay,Spanish": 0.3465909090909091, "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.26704545454545453, "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.25, "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2727272727272727, "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.3181818181818182, "Filipino,Chinese,Indonesian,Malay,Spanish": 0.30113636363636365, "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2727272727272727 }, "6_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.23863636363636365, "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.23863636363636365, "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.25, "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.2897727272727273, "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2556818181818182, "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.24431818181818182 }, "7_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.23295454545454544 } }, "AC3_2": 0.5479003849634371, "AC3_3": 0.46442064424123597, "AC3_4": 0.4102196919912001, "AC3_5": 0.3727626819026909, "AC3_6": 0.3443314825484767, "AC3_7": 0.32056571141952267 }, "prompt_5": { "overall_acc": 0.49025974025974023, "language_acc": { "English": 0.5454545454545454, "Filipino": 0.4090909090909091, "Vietnamese": 0.48295454545454547, "Chinese": 0.5397727272727273, "Indonesian": 0.42613636363636365, "Malay": 0.4715909090909091, "Spanish": 0.5568181818181818 }, "consistency_score_2": 0.5622294372294371, "consistency_score_3": 0.39285714285714296, "consistency_score_4": 0.30909090909090897, "consistency_score_5": 0.26136363636363635, "consistency_score_6": 0.23133116883116883, "consistency_score_7": 0.21022727272727273, "detailed_consistency_score": { "2_combine": { "English,Filipino": 0.48863636363636365, "English,Vietnamese": 0.5852272727272727, "English,Chinese": 0.5625, "English,Indonesian": 0.5738636363636364, "English,Malay": 0.6193181818181818, "English,Spanish": 0.7045454545454546, "Filipino,Vietnamese": 0.5340909090909091, "Filipino,Chinese": 0.5113636363636364, "Filipino,Indonesian": 0.5340909090909091, "Filipino,Malay": 0.5568181818181818, "Filipino,Spanish": 0.5056818181818182, "Vietnamese,Chinese": 0.5113636363636364, "Vietnamese,Indonesian": 0.48863636363636365, "Vietnamese,Malay": 0.5909090909090909, "Vietnamese,Spanish": 0.5738636363636364, "Chinese,Indonesian": 0.5397727272727273, "Chinese,Malay": 0.5511363636363636, "Chinese,Spanish": 0.5511363636363636, "Indonesian,Malay": 0.625, "Indonesian,Spanish": 0.6022727272727273, "Malay,Spanish": 0.5965909090909091 }, "3_combine": { "English,Filipino,Vietnamese": 0.35795454545454547, "English,Filipino,Chinese": 0.3409090909090909, "English,Filipino,Indonesian": 0.3465909090909091, "English,Filipino,Malay": 0.3693181818181818, "English,Filipino,Spanish": 0.38636363636363635, "English,Vietnamese,Chinese": 0.3693181818181818, "English,Vietnamese,Indonesian": 0.3806818181818182, "English,Vietnamese,Malay": 0.4375, "English,Vietnamese,Spanish": 0.4659090909090909, "English,Chinese,Indonesian": 0.39204545454545453, "English,Chinese,Malay": 0.3977272727272727, "English,Chinese,Spanish": 0.44886363636363635, "English,Indonesian,Malay": 0.44886363636363635, "English,Indonesian,Spanish": 0.4715909090909091, "English,Malay,Spanish": 0.48863636363636365, "Filipino,Vietnamese,Chinese": 0.3465909090909091, "Filipino,Vietnamese,Indonesian": 0.3465909090909091, "Filipino,Vietnamese,Malay": 0.39204545454545453, "Filipino,Vietnamese,Spanish": 0.36363636363636365, "Filipino,Chinese,Indonesian": 0.3465909090909091, "Filipino,Chinese,Malay": 0.375, "Filipino,Chinese,Spanish": 0.3352272727272727, "Filipino,Indonesian,Malay": 0.4034090909090909, "Filipino,Indonesian,Spanish": 0.375, "Filipino,Malay,Spanish": 0.3693181818181818, "Vietnamese,Chinese,Indonesian": 0.3352272727272727, "Vietnamese,Chinese,Malay": 0.36363636363636365, "Vietnamese,Chinese,Spanish": 0.38636363636363635, "Vietnamese,Indonesian,Malay": 0.3977272727272727, "Vietnamese,Indonesian,Spanish": 0.3977272727272727, "Vietnamese,Malay,Spanish": 0.42045454545454547, "Chinese,Indonesian,Malay": 0.4147727272727273, "Chinese,Indonesian,Spanish": 0.42045454545454547, "Chinese,Malay,Spanish": 0.39204545454545453, "Indonesian,Malay,Spanish": 0.4659090909090909 }, "4_combine": { "English,Filipino,Vietnamese,Chinese": 0.26704545454545453, "English,Filipino,Vietnamese,Indonesian": 0.2840909090909091, "English,Filipino,Vietnamese,Malay": 0.30113636363636365, "English,Filipino,Vietnamese,Spanish": 0.3125, "English,Filipino,Chinese,Indonesian": 0.2727272727272727, "English,Filipino,Chinese,Malay": 0.2784090909090909, "English,Filipino,Chinese,Spanish": 0.2840909090909091, "English,Filipino,Indonesian,Malay": 0.30113636363636365, "English,Filipino,Indonesian,Spanish": 0.3068181818181818, "English,Filipino,Malay,Spanish": 0.3068181818181818, "English,Vietnamese,Chinese,Indonesian": 0.2840909090909091, "English,Vietnamese,Chinese,Malay": 0.2897727272727273, "English,Vietnamese,Chinese,Spanish": 0.3352272727272727, "English,Vietnamese,Indonesian,Malay": 0.32954545454545453, "English,Vietnamese,Indonesian,Spanish": 0.3409090909090909, "English,Vietnamese,Malay,Spanish": 0.35795454545454547, "English,Chinese,Indonesian,Malay": 0.3352272727272727, "English,Chinese,Indonesian,Spanish": 0.36363636363636365, "English,Chinese,Malay,Spanish": 0.3465909090909091, "English,Indonesian,Malay,Spanish": 0.39204545454545453, "Filipino,Vietnamese,Chinese,Indonesian": 0.2727272727272727, "Filipino,Vietnamese,Chinese,Malay": 0.2840909090909091, "Filipino,Vietnamese,Chinese,Spanish": 0.2727272727272727, "Filipino,Vietnamese,Indonesian,Malay": 0.30113636363636365, "Filipino,Vietnamese,Indonesian,Spanish": 0.30113636363636365, "Filipino,Vietnamese,Malay,Spanish": 0.3068181818181818, "Filipino,Chinese,Indonesian,Malay": 0.30113636363636365, "Filipino,Chinese,Indonesian,Spanish": 0.29545454545454547, "Filipino,Chinese,Malay,Spanish": 0.2727272727272727, "Filipino,Indonesian,Malay,Spanish": 0.3125, "Vietnamese,Chinese,Indonesian,Malay": 0.29545454545454547, "Vietnamese,Chinese,Indonesian,Spanish": 0.3125, "Vietnamese,Chinese,Malay,Spanish": 0.3068181818181818, "Vietnamese,Indonesian,Malay,Spanish": 0.3409090909090909, "Chinese,Indonesian,Malay,Spanish": 0.3522727272727273 }, "5_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian": 0.23863636363636365, "English,Filipino,Vietnamese,Chinese,Malay": 0.23295454545454544, "English,Filipino,Vietnamese,Chinese,Spanish": 0.24431818181818182, "English,Filipino,Vietnamese,Indonesian,Malay": 0.2556818181818182, "English,Filipino,Vietnamese,Indonesian,Spanish": 0.26704545454545453, "English,Filipino,Vietnamese,Malay,Spanish": 0.26704545454545453, "English,Filipino,Chinese,Indonesian,Malay": 0.24431818181818182, "English,Filipino,Chinese,Indonesian,Spanish": 0.26136363636363635, "English,Filipino,Chinese,Malay,Spanish": 0.24431818181818182, "English,Filipino,Indonesian,Malay,Spanish": 0.2727272727272727, "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, "English,Vietnamese,Chinese,Indonesian,Spanish": 0.2784090909090909, "English,Vietnamese,Chinese,Malay,Spanish": 0.26704545454545453, "English,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, "English,Chinese,Indonesian,Malay,Spanish": 0.32386363636363635, "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.24431818181818182, "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.2556818181818182, "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.23863636363636365, "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.26704545454545453, "Filipino,Chinese,Indonesian,Malay,Spanish": 0.2556818181818182, "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2784090909090909 }, "6_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.2159090909090909, "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.23295454545454544, "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2159090909090909, "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.23863636363636365, "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.23863636363636365, "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.25, "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.22727272727272727 }, "7_combine": { "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.21022727272727273 } }, "AC3_2": 0.5237839281043418, "AC3_3": 0.43618697474052426, "AC3_4": 0.37914481938983285, "AC3_5": 0.3409581778458603, "AC3_6": 0.31433976592725005, "AC3_7": 0.29426946166661816 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6504854368932039 }, "prompt_2": { "accuracy": 0.6213592233009708 }, "prompt_3": { "accuracy": 0.6407766990291263 }, "prompt_4": { "accuracy": 0.6699029126213593 }, "prompt_5": { "accuracy": 0.6407766990291263 } }, "cn_eval": { "prompt_1": { "accuracy": 0.4666666666666667 }, "prompt_2": { "accuracy": 0.4666666666666667 }, "prompt_3": { "accuracy": 0.47619047619047616 }, "prompt_4": { "accuracy": 0.49523809523809526 }, "prompt_5": { "accuracy": 0.5142857142857142 } }, "us_eval": { "prompt_1": { "accuracy": 0.7570093457943925 }, "prompt_2": { "accuracy": 0.7476635514018691 }, "prompt_3": { "accuracy": 0.7570093457943925 }, "prompt_4": { "accuracy": 0.7476635514018691 }, "prompt_5": { "accuracy": 0.7383177570093458 } }, "ph_eval": { "prompt_1": { "accuracy": 0.61, "category_acc": { "brand": 0.5, "demographics": 0.2, "biology": 0.7, "history": 0.4666666666666667, "literature": 0.6, "politics": 0.9, "culture": 0.8, "film": 0.5, "law": 0.6, "geography": 0.7 } }, "prompt_2": { "accuracy": 0.58, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.8, "history": 0.4666666666666667, "literature": 0.5, "politics": 0.9, "culture": 0.8, "film": 0.4, "law": 0.6, "geography": 0.7 } }, "prompt_3": { "accuracy": 0.55, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.7, "history": 0.4, "literature": 0.5, "politics": 0.8, "culture": 0.7, "film": 0.6, "law": 0.6, "geography": 0.6 } }, "prompt_4": { "accuracy": 0.6, "category_acc": { "brand": 0.4, "demographics": 0.4, "biology": 0.7, "history": 0.5333333333333333, "literature": 0.4, "politics": 0.8, "culture": 0.8, "film": 0.6, "law": 0.7, "geography": 0.6 } }, "prompt_5": { "accuracy": 0.58, "category_acc": { "brand": 0.4, "demographics": 0.2, "biology": 0.8, "history": 0.5333333333333333, "literature": 0.5, "politics": 0.8, "culture": 0.6, "film": 0.4, "law": 0.6, "geography": 0.8 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.2794452574404052 }, "prompt_2": { "bleu_score": 0.2674775026836445 }, "prompt_3": { "bleu_score": 0.2722339627774794 }, "prompt_4": { "bleu_score": 0.26992687523049974 }, "prompt_5": { "bleu_score": 0.2339828457497402 } }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": { "accuracy": 0.6721120186697783 }, "prompt_2": { "accuracy": 0.6779463243873979 }, "prompt_3": { "accuracy": 0.661610268378063 }, "prompt_4": { "accuracy": 0.6779463243873979 }, "prompt_5": { "accuracy": 0.6837806301050176 } }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "meta_llama_3_70b": { "model_size": "70B", "model_link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9527310924369747, "language_acc": { "Spanish": 0.9495798319327731, "English": 0.9621848739495799, "Chinese": 0.9478991596638655, "Vietnamese": 0.9512605042016806 }, "consistency_score_2": 0.9488795518207284, "consistency_score_3": 0.9245798319327732, "consistency_score_4": 0.9058823529411765, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9613445378151261, "Spanish,Chinese": 0.9411764705882353, "Spanish,Vietnamese": 0.946218487394958, "English,Chinese": 0.9487394957983193, "English,Vietnamese": 0.9529411764705882, "Chinese,Vietnamese": 0.9428571428571428 }, "3_combine": { "Spanish,English,Chinese": 0.926890756302521, "Spanish,English,Vietnamese": 0.9310924369747899, "Spanish,Chinese,Vietnamese": 0.9168067226890756, "English,Chinese,Vietnamese": 0.9235294117647059 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9058823529411765 } }, "AC3_2": 0.9508014216050183, "AC3_3": 0.9384443906875058, "AC3_4": 0.9287162813082854 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.7457142857142858, "language_acc": { "Filipino": 0.7333333333333333, "Vietnamese": 0.7333333333333333, "Chinese": 0.7466666666666667, "Spanish": 0.76, "Malay": 0.7, "Indonesian": 0.7266666666666667, "English": 0.82 }, "consistency_score_2": 0.82, "consistency_score_3": 0.7436190476190475, "consistency_score_4": 0.6923809523809522, "consistency_score_5": 0.6517460317460317, "consistency_score_6": 0.6171428571428572, "consistency_score_7": 0.5866666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.8066666666666666, "Filipino,Chinese": 0.7933333333333333, "Filipino,Spanish": 0.7933333333333333, "Filipino,Malay": 0.78, "Filipino,Indonesian": 0.7733333333333333, "Filipino,English": 0.7866666666666666, "Vietnamese,Chinese": 0.8333333333333334, "Vietnamese,Spanish": 0.86, "Vietnamese,Malay": 0.8333333333333334, "Vietnamese,Indonesian": 0.8333333333333334, "Vietnamese,English": 0.8333333333333334, "Chinese,Spanish": 0.82, "Chinese,Malay": 0.82, "Chinese,Indonesian": 0.8, "Chinese,English": 0.8266666666666667, "Spanish,Malay": 0.8333333333333334, "Spanish,Indonesian": 0.8266666666666667, "Spanish,English": 0.8866666666666667, "Malay,Indonesian": 0.8266666666666667, "Malay,English": 0.82, "Indonesian,English": 0.8333333333333334 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.7333333333333333, "Filipino,Vietnamese,Spanish": 0.7466666666666667, "Filipino,Vietnamese,Malay": 0.7266666666666667, "Filipino,Vietnamese,Indonesian": 0.72, "Filipino,Vietnamese,English": 0.7266666666666667, "Filipino,Chinese,Spanish": 0.7333333333333333, "Filipino,Chinese,Malay": 0.7066666666666667, "Filipino,Chinese,Indonesian": 0.7, "Filipino,Chinese,English": 0.72, "Filipino,Spanish,Malay": 0.7266666666666667, "Filipino,Spanish,Indonesian": 0.7266666666666667, "Filipino,Spanish,English": 0.7533333333333333, "Filipino,Malay,Indonesian": 0.7133333333333334, "Filipino,Malay,English": 0.7066666666666667, "Filipino,Indonesian,English": 0.7266666666666667, "Vietnamese,Chinese,Spanish": 0.7666666666666667, "Vietnamese,Chinese,Malay": 0.7466666666666667, "Vietnamese,Chinese,Indonesian": 0.74, "Vietnamese,Chinese,English": 0.7533333333333333, "Vietnamese,Spanish,Malay": 0.7733333333333333, "Vietnamese,Spanish,Indonesian": 0.78, "Vietnamese,Spanish,English": 0.8, "Vietnamese,Malay,Indonesian": 0.7533333333333333, "Vietnamese,Malay,English": 0.7466666666666667, "Vietnamese,Indonesian,English": 0.7666666666666667, "Chinese,Spanish,Malay": 0.7466666666666667, "Chinese,Spanish,Indonesian": 0.74, "Chinese,Spanish,English": 0.7733333333333333, "Chinese,Malay,Indonesian": 0.7266666666666667, "Chinese,Malay,English": 0.74, "Chinese,Indonesian,English": 0.74, "Spanish,Malay,Indonesian": 0.76, "Spanish,Malay,English": 0.7733333333333333, "Spanish,Indonesian,English": 0.78, "Malay,Indonesian,English": 0.7533333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.6933333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.68, "Filipino,Vietnamese,Chinese,Indonesian": 0.6666666666666666, "Filipino,Vietnamese,Chinese,English": 0.6733333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.7, "Filipino,Vietnamese,Spanish,Indonesian": 0.7, "Filipino,Vietnamese,Spanish,English": 0.7133333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.6733333333333333, "Filipino,Vietnamese,Malay,English": 0.6733333333333333, "Filipino,Vietnamese,Indonesian,English": 0.6866666666666666, "Filipino,Chinese,Spanish,Malay": 0.68, "Filipino,Chinese,Spanish,Indonesian": 0.6733333333333333, "Filipino,Chinese,Spanish,English": 0.7, "Filipino,Chinese,Malay,Indonesian": 0.6466666666666666, "Filipino,Chinese,Malay,English": 0.6533333333333333, "Filipino,Chinese,Indonesian,English": 0.6666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.68, "Filipino,Spanish,Malay,English": 0.6933333333333334, "Filipino,Spanish,Indonesian,English": 0.7, "Filipino,Malay,Indonesian,English": 0.6666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.7066666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.7066666666666667, "Vietnamese,Chinese,Spanish,English": 0.7266666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.6866666666666666, "Vietnamese,Chinese,Malay,English": 0.6866666666666666, "Vietnamese,Chinese,Indonesian,English": 0.6933333333333334, "Vietnamese,Spanish,Malay,Indonesian": 0.7266666666666667, "Vietnamese,Spanish,Malay,English": 0.7266666666666667, "Vietnamese,Spanish,Indonesian,English": 0.74, "Vietnamese,Malay,Indonesian,English": 0.7066666666666667, "Chinese,Spanish,Malay,Indonesian": 0.6933333333333334, "Chinese,Spanish,Malay,English": 0.7, "Chinese,Spanish,Indonesian,English": 0.7066666666666667, "Chinese,Malay,Indonesian,English": 0.68, "Spanish,Malay,Indonesian,English": 0.7266666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.6533333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.6466666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.66, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.6266666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.6266666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.6333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.66, "Filipino,Vietnamese,Spanish,Malay,English": 0.6666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.6733333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.64, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.6333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.6466666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.6466666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.6133333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.6533333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.6666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.6666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.6733333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.6466666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.6933333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.66 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.6133333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.62, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.62, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.5933333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.6333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.6066666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.6333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.5866666666666667 } }, "AC3_2": 0.7810948904610614, "AC3_3": 0.7446651927952671, "AC3_4": 0.7180586565252817, "AC3_5": 0.695570844371593, "AC3_6": 0.6753638813520623, "AC3_7": 0.6566976411229783 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.6493506493506492, "language_acc": { "Indonesian": 0.625, "English": 0.7045454545454546, "Filipino": 0.5568181818181818, "Spanish": 0.6590909090909091, "Chinese": 0.7159090909090909, "Malay": 0.6534090909090909, "Vietnamese": 0.6306818181818182 }, "consistency_score_2": 0.7169913419913418, "consistency_score_3": 0.5951298701298702, "consistency_score_4": 0.5238636363636363, "consistency_score_5": 0.4751082251082251, "consistency_score_6": 0.43993506493506496, "consistency_score_7": 0.4147727272727273, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7159090909090909, "Indonesian,Filipino": 0.7159090909090909, "Indonesian,Spanish": 0.7897727272727273, "Indonesian,Chinese": 0.6875, "Indonesian,Malay": 0.7954545454545454, "Indonesian,Vietnamese": 0.7045454545454546, "English,Filipino": 0.6590909090909091, "English,Spanish": 0.75, "English,Chinese": 0.7215909090909091, "English,Malay": 0.7272727272727273, "English,Vietnamese": 0.7443181818181818, "Filipino,Spanish": 0.7329545454545454, "Filipino,Chinese": 0.6079545454545454, "Filipino,Malay": 0.7272727272727273, "Filipino,Vietnamese": 0.6647727272727273, "Spanish,Chinese": 0.6818181818181818, "Spanish,Malay": 0.8011363636363636, "Spanish,Vietnamese": 0.7272727272727273, "Chinese,Malay": 0.7159090909090909, "Chinese,Vietnamese": 0.6704545454545454, "Malay,Vietnamese": 0.7159090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.5681818181818182, "Indonesian,English,Spanish": 0.6420454545454546, "Indonesian,English,Chinese": 0.5852272727272727, "Indonesian,English,Malay": 0.6477272727272727, "Indonesian,English,Vietnamese": 0.6022727272727273, "Indonesian,Filipino,Spanish": 0.6363636363636364, "Indonesian,Filipino,Chinese": 0.5397727272727273, "Indonesian,Filipino,Malay": 0.6477272727272727, "Indonesian,Filipino,Vietnamese": 0.5681818181818182, "Indonesian,Spanish,Chinese": 0.5965909090909091, "Indonesian,Spanish,Malay": 0.7272727272727273, "Indonesian,Spanish,Vietnamese": 0.625, "Indonesian,Chinese,Malay": 0.6193181818181818, "Indonesian,Chinese,Vietnamese": 0.5625, "Indonesian,Malay,Vietnamese": 0.6306818181818182, "English,Filipino,Spanish": 0.5795454545454546, "English,Filipino,Chinese": 0.5227272727272727, "English,Filipino,Malay": 0.5738636363636364, "English,Filipino,Vietnamese": 0.5397727272727273, "English,Spanish,Chinese": 0.5909090909090909, "English,Spanish,Malay": 0.6477272727272727, "English,Spanish,Vietnamese": 0.625, "English,Chinese,Malay": 0.5965909090909091, "English,Chinese,Vietnamese": 0.5965909090909091, "English,Malay,Vietnamese": 0.6136363636363636, "Filipino,Spanish,Chinese": 0.5397727272727273, "Filipino,Spanish,Malay": 0.6420454545454546, "Filipino,Spanish,Vietnamese": 0.5738636363636364, "Filipino,Chinese,Malay": 0.5454545454545454, "Filipino,Chinese,Vietnamese": 0.4943181818181818, "Filipino,Malay,Vietnamese": 0.5625, "Spanish,Chinese,Malay": 0.6136363636363636, "Spanish,Chinese,Vietnamese": 0.5625, "Spanish,Malay,Vietnamese": 0.6363636363636364, "Chinese,Malay,Vietnamese": 0.5738636363636364 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.5284090909090909, "Indonesian,English,Filipino,Chinese": 0.4772727272727273, "Indonesian,English,Filipino,Malay": 0.5397727272727273, "Indonesian,English,Filipino,Vietnamese": 0.4943181818181818, "Indonesian,English,Spanish,Chinese": 0.5227272727272727, "Indonesian,English,Spanish,Malay": 0.6079545454545454, "Indonesian,English,Spanish,Vietnamese": 0.5625, "Indonesian,English,Chinese,Malay": 0.5340909090909091, "Indonesian,English,Chinese,Vietnamese": 0.5284090909090909, "Indonesian,English,Malay,Vietnamese": 0.5738636363636364, "Indonesian,Filipino,Spanish,Chinese": 0.5, "Indonesian,Filipino,Spanish,Malay": 0.5965909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.5284090909090909, "Indonesian,Filipino,Chinese,Malay": 0.5170454545454546, "Indonesian,Filipino,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.5284090909090909, "Indonesian,Spanish,Chinese,Malay": 0.5738636363636364, "Indonesian,Spanish,Chinese,Vietnamese": 0.5170454545454546, "Indonesian,Spanish,Malay,Vietnamese": 0.5909090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.5284090909090909, "English,Filipino,Spanish,Chinese": 0.48863636363636365, "English,Filipino,Spanish,Malay": 0.5397727272727273, "English,Filipino,Spanish,Vietnamese": 0.4943181818181818, "English,Filipino,Chinese,Malay": 0.4772727272727273, "English,Filipino,Chinese,Vietnamese": 0.4659090909090909, "English,Filipino,Malay,Vietnamese": 0.4943181818181818, "English,Spanish,Chinese,Malay": 0.5340909090909091, "English,Spanish,Chinese,Vietnamese": 0.5340909090909091, "English,Spanish,Malay,Vietnamese": 0.5738636363636364, "English,Chinese,Malay,Vietnamese": 0.5340909090909091, "Filipino,Spanish,Chinese,Malay": 0.5056818181818182, "Filipino,Spanish,Chinese,Vietnamese": 0.4602272727272727, "Filipino,Spanish,Malay,Vietnamese": 0.5284090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.4602272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.5284090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.44886363636363635, "Indonesian,English,Filipino,Spanish,Malay": 0.5113636363636364, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.4715909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.45454545454545453, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4431818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.4772727272727273, "Indonesian,English,Spanish,Chinese,Malay": 0.5, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4943181818181818, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5454545454545454, "Indonesian,English,Chinese,Malay,Vietnamese": 0.5, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.48295454545454547, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.5, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.44886363636363635, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.5, "English,Filipino,Spanish,Chinese,Malay": 0.45454545454545453, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.4375, "English,Spanish,Chinese,Malay,Vietnamese": 0.5, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4431818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.4318181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4602272727272727, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4318181818181818, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4147727272727273 } }, "AC3_2": 0.6814967210508039, "AC3_3": 0.6210590869627431, "AC3_4": 0.5798960966586238, "AC3_5": 0.5487294225978513, "AC3_6": 0.524512745030912, "AC3_7": 0.5062062269824554 } }, "sg_eval": { "prompt_1": { "accuracy": 0.7864077669902912 } }, "cn_eval": { "prompt_1": { "accuracy": 0.5523809523809524 } }, "us_eval": { "prompt_1": { "accuracy": 0.8691588785046729 } }, "ph_eval": { "prompt_1": { "accuracy": 0.72, "category_acc": { "brand": 0.6, "demographics": 0.4, "biology": 0.9, "history": 0.6, "literature": 0.6, "politics": 0.9, "culture": 1.0, "film": 0.7, "law": 0.9, "geography": 0.5 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.392216616269553 } }, "indommlu": { "prompt_1": { "accuracy": 0.6639295013018226, "category_acc": { "History": 0.7028112449799196, "Geography": 0.6551020408163265, "Lampungic": 0.3945578231292517, "Social science": 0.8831385642737897, "Balinese": 0.4012738853503185, "Makassarese": 0.44623655913978494, "Banjarese": 0.5208333333333334, "Chemistry": 0.4656934306569343, "Biology": 0.6911242603550296, "Science": 0.8328173374613003, "Christian religion": 0.7960199004975125, "Art": 0.7504159733777038, "Islam religion": 0.8051209103840683, "Hindu religion": 0.6733333333333333, "Madurese": 0.3559322033898305, "Sport": 0.7094594594594594, "Indonesian language": 0.7023661270236613, "Physics": 0.5656565656565656, "Minangkabau culture": 0.49748743718592964, "Dayak language": 0.4036697247706422, "Sociology": 0.6532258064516129, "Economy": 0.6516393442622951, "Sundanese": 0.6041486603284356, "Javanese": 0.5806451612903226, "Civic education": 0.7811158798283262 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.44668974188236693 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.37308599924167435 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.2950082874458052 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.4633895225462568 } }, "mmlu": { "prompt_1": { "accuracy": 0.7607934655775963 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.7634608509116911, "category_acc": { "high_school_european_history": 0.8170731707317073, "business_ethics": 0.7878787878787878, "clinical_knowledge": 0.8446969696969697, "medical_genetics": 0.8686868686868687, "high_school_us_history": 0.9458128078817734, "high_school_physics": 0.58, "high_school_world_history": 0.9110169491525424, "virology": 0.5454545454545454, "high_school_microeconomics": 0.9029535864978903, "econometrics": 0.6902654867256637, "college_computer_science": 0.6868686868686869, "high_school_biology": 0.889967637540453, "abstract_algebra": 0.43434343434343436, "professional_accounting": 0.604982206405694, "philosophy": 0.8290322580645161, "professional_medicine": 0.8597785977859779, "nutrition": 0.8491803278688524, "global_facts": 0.5252525252525253, "machine_learning": 0.6756756756756757, "security_studies": 0.7909836065573771, "public_relations": 0.7431192660550459, "professional_psychology": 0.8477905073649754, "prehistory": 0.9009287925696594, "anatomy": 0.7835820895522388, "human_sexuality": 0.8538461538461538, "college_medicine": 0.75, "high_school_government_and_politics": 0.9791666666666666, "college_chemistry": 0.5959595959595959, "logical_fallacies": 0.8518518518518519, "high_school_geography": 0.9137055837563451, "elementary_mathematics": 0.6206896551724138, "human_aging": 0.7927927927927928, "college_mathematics": 0.5151515151515151, "high_school_psychology": 0.9319852941176471, "formal_logic": 0.568, "high_school_statistics": 0.6976744186046512, "international_law": 0.8583333333333333, "high_school_mathematics": 0.4423791821561338, "high_school_computer_science": 0.8484848484848485, "conceptual_physics": 0.782051282051282, "miscellaneous": 0.9092071611253197, "high_school_chemistry": 0.7277227722772277, "marketing": 0.9356223175965666, "professional_law": 0.6073059360730594, "management": 0.9019607843137255, "college_physics": 0.5346534653465347, "jurisprudence": 0.8878504672897196, "world_religions": 0.8941176470588236, "sociology": 0.915, "us_foreign_policy": 0.9090909090909091, "high_school_macroeconomics": 0.8303341902313625, "computer_security": 0.8282828282828283, "moral_scenarios": 0.4899328859060403, "moral_disputes": 0.8376811594202899, "electrical_engineering": 0.7361111111111112, "astronomy": 0.9006622516556292, "college_biology": 0.9090909090909091 } } }, "c_eval": { "prompt_1": { "accuracy": 0.6552748885586924 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.6494396014943961, "category_acc": { "computer_network": 0.7083333333333334, "operating_system": 0.7916666666666666, "computer_architecture": 0.6923076923076923, "college_programming": 0.6904761904761905, "college_physics": 0.5, "college_chemistry": 0.4827586206896552, "advanced_mathematics": 0.625, "probability_and_statistics": 0.5652173913043478, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.5476190476190477, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.5416666666666666, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.625, "middle_school_mathematics": 0.5416666666666666, "middle_school_biology": 0.9230769230769231, "middle_school_physics": 0.75, "middle_school_chemistry": 0.88, "veterinary_medicine": 0.75, "college_economics": 0.6166666666666667, "business_administration": 0.6052631578947368, "marxism": 0.7916666666666666, "mao_zedong_thought": 0.7586206896551724, "education_science": 0.5882352941176471, "teacher_qualification": 0.8367346938775511, "high_school_politics": 0.875, "high_school_geography": 0.75, "middle_school_politics": 0.8846153846153846, "middle_school_geography": 0.8823529411764706, "modern_chinese_history": 0.7142857142857143, "ideological_and_moral_cultivation": 0.8333333333333334, "logic": 0.5925925925925926, "law": 0.5862068965517241, "chinese_language_and_literature": 0.6428571428571429, "art_studies": 0.7368421052631579, "professional_tour_guide": 0.6470588235294118, "legal_professional": 0.5357142857142857, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.72, "middle_school_history": 0.8518518518518519, "civil_servant": 0.5769230769230769, "sports_science": 0.5833333333333334, "plant_protection": 0.8148148148148148, "basic_medicine": 0.6666666666666666, "clinical_medicine": 0.5185185185185185, "urban_and_rural_planner": 0.5490196078431373, "accountant": 0.6851851851851852, "fire_engineer": 0.5, "environmental_impact_assessment_engineer": 0.6388888888888888, "tax_accountant": 0.5555555555555556, "physician": 0.7037037037037037 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.6881720430107527 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.6723363840442065, "category_acc": { "agronomy": 0.6804733727810651, "anatomy": 0.5337837837837838, "ancient_chinese": 0.3780487804878049, "arts": 0.78125, "astronomy": 0.49696969696969695, "business_ethics": 0.6842105263157895, "chinese_civil_service_exam": 0.6125, "chinese_driving_rule": 0.7709923664122137, "chinese_food_culture": 0.5735294117647058, "chinese_foreign_policy": 0.7757009345794392, "chinese_history": 0.7770897832817337, "chinese_literature": 0.45098039215686275, "chinese_teacher_qualification": 0.7932960893854749, "clinical_knowledge": 0.6455696202531646, "college_actuarial_science": 0.44339622641509435, "college_education": 0.794392523364486, "college_engineering_hydrology": 0.6415094339622641, "college_law": 0.5462962962962963, "college_mathematics": 0.4, "college_medical_statistics": 0.7169811320754716, "college_medicine": 0.717948717948718, "computer_science": 0.7401960784313726, "computer_security": 0.847953216374269, "conceptual_physics": 0.7074829931972789, "construction_project_management": 0.4748201438848921, "economics": 0.8113207547169812, "education": 0.656441717791411, "electrical_engineering": 0.6511627906976745, "elementary_chinese": 0.5714285714285714, "elementary_commonsense": 0.6767676767676768, "elementary_information_and_technology": 0.9243697478991597, "elementary_mathematics": 0.48695652173913045, "ethnology": 0.725925925925926, "food_science": 0.6363636363636364, "genetics": 0.6306818181818182, "global_facts": 0.7785234899328859, "high_school_biology": 0.5502958579881657, "high_school_chemistry": 0.48484848484848486, "high_school_geography": 0.6694915254237288, "high_school_mathematics": 0.3597560975609756, "high_school_physics": 0.5727272727272728, "high_school_politics": 0.6783216783216783, "human_sexuality": 0.6904761904761905, "international_law": 0.6324324324324324, "journalism": 0.6802325581395349, "jurisprudence": 0.6934306569343066, "legal_and_moral_basis": 0.9205607476635514, "logical": 0.7560975609756098, "machine_learning": 0.680327868852459, "management": 0.8047619047619048, "marketing": 0.7666666666666667, "marxist_theory": 0.7566137566137566, "modern_chinese": 0.4827586206896552, "nutrition": 0.6482758620689655, "philosophy": 0.6952380952380952, "professional_accounting": 0.7314285714285714, "professional_law": 0.5402843601895735, "professional_medicine": 0.6675531914893617, "professional_psychology": 0.7801724137931034, "public_relations": 0.6494252873563219, "security_study": 0.7703703703703704, "sociology": 0.7168141592920354, "sports_science": 0.7090909090909091, "traditional_chinese_medicine": 0.5945945945945946, "virology": 0.7455621301775148, "world_history": 0.7701863354037267, "world_religions": 0.83125 } } }, "zbench": { "prompt_1": { "accuracy": 0.45454545454545453 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.7295454545454545 } }, "ocnli": { "prompt_1": { "accuracy": 0.7122033898305085 } }, "c3": { "prompt_1": { "accuracy": 0.9427823485415109 } }, "dream": { "prompt_1": { "accuracy": 0.9485546300832925 } }, "samsum": { "prompt_1": { "rouge1": 0.496295637878035, "rouge2": 0.25307430676703274, "rougeL": 0.4104483122091349, "avg_rouge": 0.38660608561806753 } }, "dialogsum": { "prompt_1": { "rouge1": 0.45314028137020645, "rouge2": 0.20825564368292146, "rougeL": 0.3797842726194882, "avg_rouge": 0.34706006589087207 } }, "sst2": { "prompt_1": { "accuracy": 0.9243119266055045 } }, "cola": { "prompt_1": { "accuracy": 0.7526366251198466 } }, "qqp": { "prompt_1": { "accuracy": 0.7485 } }, "mnli": { "prompt_1": { "accuracy": 0.6 } }, "qnli": { "prompt_1": { "accuracy": 0.6 } }, "wnli": { "prompt_1": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 0.7 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 } } } }, "meta_llama_3_70b_instruct": { "model_size": "70B", "model_link": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9552521008403362, "language_acc": { "Spanish": 0.9554621848739496, "English": 0.9647058823529412, "Chinese": 0.9420168067226891, "Vietnamese": 0.9588235294117647 }, "consistency_score_2": 0.9554621848739496, "consistency_score_3": 0.9352941176470588, "consistency_score_4": 0.9210084033613445, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9672268907563025, "Spanish,Chinese": 0.9411764705882353, "Spanish,Vietnamese": 0.9588235294117647, "English,Chinese": 0.9537815126050421, "English,Vietnamese": 0.9621848739495799, "Chinese,Vietnamese": 0.9495798319327731 }, "3_combine": { "Spanish,English,Chinese": 0.9327731092436975, "Spanish,English,Vietnamese": 0.9453781512605042, "Spanish,Chinese,Vietnamese": 0.9277310924369748, "English,Chinese,Vietnamese": 0.9352941176470588 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9210084033613445 } }, "AC3_2": 0.9553571312577183, "AC3_3": 0.9451677636884602, "AC3_4": 0.9378177605779494 }, "prompt_2": { "overall_acc": 0.957983193277311, "language_acc": { "Spanish": 0.9571428571428572, "English": 0.9647058823529412, "Chinese": 0.9478991596638655, "Vietnamese": 0.9621848739495799 }, "consistency_score_2": 0.9557422969187676, "consistency_score_3": 0.9348739495798319, "consistency_score_4": 0.9193277310924369, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9621848739495799, "Spanish,Chinese": 0.9445378151260504, "Spanish,Vietnamese": 0.9621848739495799, "English,Chinese": 0.9487394957983193, "English,Vietnamese": 0.9647058823529412, "Chinese,Vietnamese": 0.9521008403361344 }, "3_combine": { "Spanish,English,Chinese": 0.9285714285714286, "Spanish,English,Vietnamese": 0.9453781512605042, "Spanish,Chinese,Vietnamese": 0.9302521008403362, "English,Chinese,Vietnamese": 0.9352941176470588 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9193277310924369 } }, "AC3_2": 0.9568614330478293, "AC3_3": 0.9462875049631115, "AC3_4": 0.9382574873675487 }, "prompt_3": { "overall_acc": 0.9554621848739496, "language_acc": { "Spanish": 0.9563025210084034, "English": 0.9663865546218487, "Chinese": 0.9411764705882353, "Vietnamese": 0.957983193277311 }, "consistency_score_2": 0.9494397759103642, "consistency_score_3": 0.9260504201680673, "consistency_score_4": 0.9100840336134454, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9596638655462185, "Spanish,Chinese": 0.9436974789915966, "Spanish,Vietnamese": 0.9512605042016806, "English,Chinese": 0.9420168067226891, "English,Vietnamese": 0.9554621848739496, "Chinese,Vietnamese": 0.9445378151260504 }, "3_combine": { "Spanish,English,Chinese": 0.9252100840336135, "Spanish,English,Vietnamese": 0.9352941176470588, "Spanish,Chinese,Vietnamese": 0.9210084033613445, "English,Chinese,Vietnamese": 0.9226890756302522 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9100840336134454 } }, "AC3_2": 0.9524414603221062, "AC3_3": 0.9405264204333463, "AC3_4": 0.9322212127594778 }, "prompt_4": { "overall_acc": 0.957983193277311, "language_acc": { "Spanish": 0.957983193277311, "English": 0.9672268907563025, "Chinese": 0.9445378151260504, "Vietnamese": 0.9621848739495799 }, "consistency_score_2": 0.961764705882353, "consistency_score_3": 0.9441176470588235, "consistency_score_4": 0.9310924369747899, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.973109243697479, "Spanish,Chinese": 0.9478991596638655, "Spanish,Vietnamese": 0.9680672268907563, "English,Chinese": 0.9588235294117647, "English,Vietnamese": 0.9714285714285714, "Chinese,Vietnamese": 0.9512605042016806 }, "3_combine": { "Spanish,English,Chinese": 0.9420168067226891, "Spanish,English,Vietnamese": 0.9571428571428572, "Spanish,Chinese,Vietnamese": 0.9352941176470588, "English,Chinese,Vietnamese": 0.9420168067226891 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9310924369747899 } }, "AC3_2": 0.9598702251247714, "AC3_3": 0.9509998830046162, "AC3_4": 0.9443464217927687 }, "prompt_5": { "overall_acc": 0.9563025210084034, "language_acc": { "Spanish": 0.9638655462184874, "English": 0.9663865546218487, "Chinese": 0.9369747899159664, "Vietnamese": 0.957983193277311 }, "consistency_score_2": 0.9547619047619048, "consistency_score_3": 0.9334033613445378, "consistency_score_4": 0.9176470588235294, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9714285714285714, "Spanish,Chinese": 0.9420168067226891, "Spanish,Vietnamese": 0.9647058823529412, "English,Chinese": 0.9428571428571428, "English,Vietnamese": 0.9663865546218487, "Chinese,Vietnamese": 0.9411764705882353 }, "3_combine": { "Spanish,English,Chinese": 0.9294117647058824, "Spanish,English,Vietnamese": 0.9521008403361344, "Spanish,Chinese,Vietnamese": 0.9252100840336135, "English,Chinese,Vietnamese": 0.926890756302521 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9176470588235294 } }, "AC3_2": 0.9555315918465609, "AC3_3": 0.9447141969128393, "AC3_4": 0.9365761012425562 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.7561904761904763, "language_acc": { "Filipino": 0.74, "Vietnamese": 0.7266666666666667, "Chinese": 0.74, "Spanish": 0.7533333333333333, "Malay": 0.72, "Indonesian": 0.7733333333333333, "English": 0.84 }, "consistency_score_2": 0.7523809523809524, "consistency_score_3": 0.6474285714285716, "consistency_score_4": 0.5834285714285715, "consistency_score_5": 0.5374603174603174, "consistency_score_6": 0.5019047619047619, "consistency_score_7": 0.47333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.7333333333333333, "Filipino,Chinese": 0.6533333333333333, "Filipino,Spanish": 0.7733333333333333, "Filipino,Malay": 0.7133333333333334, "Filipino,Indonesian": 0.7533333333333333, "Filipino,English": 0.74, "Vietnamese,Chinese": 0.7333333333333333, "Vietnamese,Spanish": 0.8, "Vietnamese,Malay": 0.7533333333333333, "Vietnamese,Indonesian": 0.78, "Vietnamese,English": 0.74, "Chinese,Spanish": 0.7666666666666667, "Chinese,Malay": 0.7, "Chinese,Indonesian": 0.7066666666666667, "Chinese,English": 0.7666666666666667, "Spanish,Malay": 0.74, "Spanish,Indonesian": 0.7866666666666666, "Spanish,English": 0.8066666666666666, "Malay,Indonesian": 0.8133333333333334, "Malay,English": 0.7333333333333333, "Indonesian,English": 0.8066666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.58, "Filipino,Vietnamese,Spanish": 0.6733333333333333, "Filipino,Vietnamese,Malay": 0.6266666666666667, "Filipino,Vietnamese,Indonesian": 0.6466666666666666, "Filipino,Vietnamese,English": 0.64, "Filipino,Chinese,Spanish": 0.6133333333333333, "Filipino,Chinese,Malay": 0.5666666666666667, "Filipino,Chinese,Indonesian": 0.58, "Filipino,Chinese,English": 0.6066666666666667, "Filipino,Spanish,Malay": 0.6466666666666666, "Filipino,Spanish,Indonesian": 0.6733333333333333, "Filipino,Spanish,English": 0.6733333333333333, "Filipino,Malay,Indonesian": 0.66, "Filipino,Malay,English": 0.62, "Filipino,Indonesian,English": 0.66, "Vietnamese,Chinese,Spanish": 0.66, "Vietnamese,Chinese,Malay": 0.62, "Vietnamese,Chinese,Indonesian": 0.62, "Vietnamese,Chinese,English": 0.6266666666666667, "Vietnamese,Spanish,Malay": 0.66, "Vietnamese,Spanish,Indonesian": 0.6933333333333334, "Vietnamese,Spanish,English": 0.6866666666666666, "Vietnamese,Malay,Indonesian": 0.6866666666666666, "Vietnamese,Malay,English": 0.64, "Vietnamese,Indonesian,English": 0.6866666666666666, "Chinese,Spanish,Malay": 0.6133333333333333, "Chinese,Spanish,Indonesian": 0.6466666666666666, "Chinese,Spanish,English": 0.68, "Chinese,Malay,Indonesian": 0.64, "Chinese,Malay,English": 0.6266666666666667, "Chinese,Indonesian,English": 0.6666666666666666, "Spanish,Malay,Indonesian": 0.6866666666666666, "Spanish,Malay,English": 0.6533333333333333, "Spanish,Indonesian,English": 0.7133333333333334, "Malay,Indonesian,English": 0.6866666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.56, "Filipino,Vietnamese,Chinese,Malay": 0.5266666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.5333333333333333, "Filipino,Vietnamese,Chinese,English": 0.5466666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.5866666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.6066666666666667, "Filipino,Vietnamese,Spanish,English": 0.6133333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.6, "Filipino,Vietnamese,Malay,English": 0.5733333333333334, "Filipino,Vietnamese,Indonesian,English": 0.5933333333333334, "Filipino,Chinese,Spanish,Malay": 0.5333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.5533333333333333, "Filipino,Chinese,Spanish,English": 0.58, "Filipino,Chinese,Malay,Indonesian": 0.5466666666666666, "Filipino,Chinese,Malay,English": 0.5333333333333333, "Filipino,Chinese,Indonesian,English": 0.5533333333333333, "Filipino,Spanish,Malay,Indonesian": 0.6066666666666667, "Filipino,Spanish,Malay,English": 0.5866666666666667, "Filipino,Spanish,Indonesian,English": 0.6133333333333333, "Filipino,Malay,Indonesian,English": 0.5933333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.5733333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.58, "Vietnamese,Chinese,Spanish,English": 0.5933333333333334, "Vietnamese,Chinese,Malay,Indonesian": 0.5866666666666667, "Vietnamese,Chinese,Malay,English": 0.5666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.58, "Vietnamese,Spanish,Malay,Indonesian": 0.6266666666666667, "Vietnamese,Spanish,Malay,English": 0.6, "Vietnamese,Spanish,Indonesian,English": 0.64, "Vietnamese,Malay,Indonesian,English": 0.6133333333333333, "Chinese,Spanish,Malay,Indonesian": 0.5933333333333334, "Chinese,Spanish,Malay,English": 0.58, "Chinese,Spanish,Indonesian,English": 0.62, "Chinese,Malay,Indonesian,English": 0.6, "Spanish,Malay,Indonesian,English": 0.6266666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.5066666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.5333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.5, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.5066666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.56, "Filipino,Vietnamese,Spanish,Malay,English": 0.5466666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.5533333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.52, "Filipino,Chinese,Spanish,Malay,English": 0.5133333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.5333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.52, "Filipino,Spanish,Malay,Indonesian,English": 0.56, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.5533333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.54, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.5533333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.5466666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.58, "Chinese,Spanish,Malay,Indonesian,English": 0.5666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.49333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.4866666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.49333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.4866666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.5266666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.5, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.5266666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.47333333333333333 } }, "AC3_2": 0.7542809042309045, "AC3_3": 0.6975957194702973, "AC3_4": 0.6586695374373748, "AC3_5": 0.6283339759958348, "AC3_6": 0.6033495547626118, "AC3_7": 0.5822256648119284 }, "prompt_2": { "overall_acc": 0.742857142857143, "language_acc": { "Filipino": 0.6866666666666666, "Vietnamese": 0.7466666666666667, "Chinese": 0.74, "Spanish": 0.7533333333333333, "Malay": 0.74, "Indonesian": 0.7466666666666667, "English": 0.7866666666666666 }, "consistency_score_2": 0.7419047619047618, "consistency_score_3": 0.6302857142857143, "consistency_score_4": 0.5601904761904761, "consistency_score_5": 0.5085714285714287, "consistency_score_6": 0.46761904761904766, "consistency_score_7": 0.43333333333333335, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6866666666666666, "Filipino,Chinese": 0.6466666666666666, "Filipino,Spanish": 0.7133333333333334, "Filipino,Malay": 0.7333333333333333, "Filipino,Indonesian": 0.7266666666666667, "Filipino,English": 0.6933333333333334, "Vietnamese,Chinese": 0.7666666666666667, "Vietnamese,Spanish": 0.8, "Vietnamese,Malay": 0.7266666666666667, "Vietnamese,Indonesian": 0.7466666666666667, "Vietnamese,English": 0.78, "Chinese,Spanish": 0.7866666666666666, "Chinese,Malay": 0.7, "Chinese,Indonesian": 0.74, "Chinese,English": 0.7866666666666666, "Spanish,Malay": 0.7, "Spanish,Indonesian": 0.7533333333333333, "Spanish,English": 0.7733333333333333, "Malay,Indonesian": 0.7933333333333333, "Malay,English": 0.7466666666666667, "Indonesian,English": 0.78 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.58, "Filipino,Vietnamese,Spanish": 0.62, "Filipino,Vietnamese,Malay": 0.5933333333333334, "Filipino,Vietnamese,Indonesian": 0.6, "Filipino,Vietnamese,English": 0.6, "Filipino,Chinese,Spanish": 0.5866666666666667, "Filipino,Chinese,Malay": 0.5533333333333333, "Filipino,Chinese,Indonesian": 0.5733333333333334, "Filipino,Chinese,English": 0.5866666666666667, "Filipino,Spanish,Malay": 0.6, "Filipino,Spanish,Indonesian": 0.62, "Filipino,Spanish,English": 0.6066666666666667, "Filipino,Malay,Indonesian": 0.6466666666666666, "Filipino,Malay,English": 0.6066666666666667, "Filipino,Indonesian,English": 0.6133333333333333, "Vietnamese,Chinese,Spanish": 0.6866666666666666, "Vietnamese,Chinese,Malay": 0.6133333333333333, "Vietnamese,Chinese,Indonesian": 0.64, "Vietnamese,Chinese,English": 0.6866666666666666, "Vietnamese,Spanish,Malay": 0.6333333333333333, "Vietnamese,Spanish,Indonesian": 0.6666666666666666, "Vietnamese,Spanish,English": 0.6933333333333334, "Vietnamese,Malay,Indonesian": 0.6533333333333333, "Vietnamese,Malay,English": 0.64, "Vietnamese,Indonesian,English": 0.6733333333333333, "Chinese,Spanish,Malay": 0.6, "Chinese,Spanish,Indonesian": 0.6533333333333333, "Chinese,Spanish,English": 0.68, "Chinese,Malay,Indonesian": 0.64, "Chinese,Malay,English": 0.6333333333333333, "Chinese,Indonesian,English": 0.6666666666666666, "Spanish,Malay,Indonesian": 0.6466666666666666, "Spanish,Malay,English": 0.6266666666666667, "Spanish,Indonesian,English": 0.6666666666666666, "Malay,Indonesian,English": 0.6733333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.54, "Filipino,Vietnamese,Chinese,Malay": 0.5, "Filipino,Vietnamese,Chinese,Indonesian": 0.52, "Filipino,Vietnamese,Chinese,English": 0.54, "Filipino,Vietnamese,Spanish,Malay": 0.54, "Filipino,Vietnamese,Spanish,Indonesian": 0.5533333333333333, "Filipino,Vietnamese,Spanish,English": 0.5533333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.56, "Filipino,Vietnamese,Malay,English": 0.54, "Filipino,Vietnamese,Indonesian,English": 0.5466666666666666, "Filipino,Chinese,Spanish,Malay": 0.5066666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.54, "Filipino,Chinese,Spanish,English": 0.54, "Filipino,Chinese,Malay,Indonesian": 0.52, "Filipino,Chinese,Malay,English": 0.5133333333333333, "Filipino,Chinese,Indonesian,English": 0.5333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.56, "Filipino,Spanish,Malay,English": 0.54, "Filipino,Spanish,Indonesian,English": 0.56, "Filipino,Malay,Indonesian,English": 0.5666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.5666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.5933333333333334, "Vietnamese,Chinese,Spanish,English": 0.6333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.5733333333333334, "Vietnamese,Chinese,Malay,English": 0.58, "Vietnamese,Chinese,Indonesian,English": 0.6, "Vietnamese,Spanish,Malay,Indonesian": 0.5933333333333334, "Vietnamese,Spanish,Malay,English": 0.5866666666666667, "Vietnamese,Spanish,Indonesian,English": 0.62, "Vietnamese,Malay,Indonesian,English": 0.5933333333333334, "Chinese,Spanish,Malay,Indonesian": 0.5666666666666667, "Chinese,Spanish,Malay,English": 0.56, "Chinese,Spanish,Indonesian,English": 0.5933333333333334, "Chinese,Malay,Indonesian,English": 0.5866666666666667, "Spanish,Malay,Indonesian,English": 0.5866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.47333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.49333333333333335, "Filipino,Vietnamese,Chinese,Spanish,English": 0.5066666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.48, "Filipino,Vietnamese,Chinese,Malay,English": 0.48, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.5, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5133333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.5133333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.4866666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.47333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.5, "Filipino,Chinese,Malay,Indonesian,English": 0.4866666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.5133333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.5333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.54, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.56, "Vietnamese,Chinese,Malay,Indonesian,English": 0.54, "Vietnamese,Spanish,Malay,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.5266666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4533333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.4533333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.48, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.5066666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 } }, "AC3_2": 0.7423806468848484, "AC3_3": 0.6819570774130317, "AC3_4": 0.6387203207060935, "AC3_5": 0.603783431132444, "AC3_6": 0.5739462739775373, "AC3_7": 0.5473684210060942 }, "prompt_3": { "overall_acc": 0.6980952380952381, "language_acc": { "Filipino": 0.66, "Vietnamese": 0.7533333333333333, "Chinese": 0.6533333333333333, "Spanish": 0.7, "Malay": 0.7133333333333334, "Indonesian": 0.7133333333333334, "English": 0.6933333333333334 }, "consistency_score_2": 0.6914285714285715, "consistency_score_3": 0.5561904761904761, "consistency_score_4": 0.47638095238095246, "consistency_score_5": 0.4228571428571429, "consistency_score_6": 0.3838095238095239, "consistency_score_7": 0.35333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6666666666666666, "Filipino,Chinese": 0.6133333333333333, "Filipino,Spanish": 0.7133333333333334, "Filipino,Malay": 0.6866666666666666, "Filipino,Indonesian": 0.68, "Filipino,English": 0.68, "Vietnamese,Chinese": 0.7133333333333334, "Vietnamese,Spanish": 0.68, "Vietnamese,Malay": 0.7333333333333333, "Vietnamese,Indonesian": 0.78, "Vietnamese,English": 0.6933333333333334, "Chinese,Spanish": 0.6666666666666666, "Chinese,Malay": 0.66, "Chinese,Indonesian": 0.7133333333333334, "Chinese,English": 0.6733333333333333, "Spanish,Malay": 0.6333333333333333, "Spanish,Indonesian": 0.7066666666666667, "Spanish,English": 0.6733333333333333, "Malay,Indonesian": 0.78, "Malay,English": 0.6533333333333333, "Indonesian,English": 0.72 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.52, "Filipino,Vietnamese,Spanish": 0.5533333333333333, "Filipino,Vietnamese,Malay": 0.5733333333333334, "Filipino,Vietnamese,Indonesian": 0.58, "Filipino,Vietnamese,English": 0.54, "Filipino,Chinese,Spanish": 0.52, "Filipino,Chinese,Malay": 0.5, "Filipino,Chinese,Indonesian": 0.52, "Filipino,Chinese,English": 0.5133333333333333, "Filipino,Spanish,Malay": 0.54, "Filipino,Spanish,Indonesian": 0.5533333333333333, "Filipino,Spanish,English": 0.5466666666666666, "Filipino,Malay,Indonesian": 0.5933333333333334, "Filipino,Malay,English": 0.54, "Filipino,Indonesian,English": 0.56, "Vietnamese,Chinese,Spanish": 0.56, "Vietnamese,Chinese,Malay": 0.5733333333333334, "Vietnamese,Chinese,Indonesian": 0.6266666666666667, "Vietnamese,Chinese,English": 0.5666666666666667, "Vietnamese,Spanish,Malay": 0.5333333333333333, "Vietnamese,Spanish,Indonesian": 0.5933333333333334, "Vietnamese,Spanish,English": 0.54, "Vietnamese,Malay,Indonesian": 0.66, "Vietnamese,Malay,English": 0.56, "Vietnamese,Indonesian,English": 0.62, "Chinese,Spanish,Malay": 0.5, "Chinese,Spanish,Indonesian": 0.5533333333333333, "Chinese,Spanish,English": 0.5266666666666666, "Chinese,Malay,Indonesian": 0.5933333333333334, "Chinese,Malay,English": 0.52, "Chinese,Indonesian,English": 0.5733333333333334, "Spanish,Malay,Indonesian": 0.5666666666666667, "Spanish,Malay,English": 0.49333333333333335, "Spanish,Indonesian,English": 0.56, "Malay,Indonesian,English": 0.5933333333333334 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.4666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.46, "Filipino,Vietnamese,Chinese,Indonesian": 0.47333333333333333, "Filipino,Vietnamese,Chinese,English": 0.4666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.4666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.4866666666666667, "Filipino,Vietnamese,Spanish,English": 0.47333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.5266666666666666, "Filipino,Vietnamese,Malay,English": 0.48, "Filipino,Vietnamese,Indonesian,English": 0.5, "Filipino,Chinese,Spanish,Malay": 0.44, "Filipino,Chinese,Spanish,Indonesian": 0.4533333333333333, "Filipino,Chinese,Spanish,English": 0.44666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.46, "Filipino,Chinese,Malay,English": 0.44, "Filipino,Chinese,Indonesian,English": 0.4666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.48, "Filipino,Spanish,Malay,English": 0.44666666666666666, "Filipino,Spanish,Indonesian,English": 0.4666666666666667, "Filipino,Malay,Indonesian,English": 0.5066666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.5, "Vietnamese,Chinese,Spanish,English": 0.4866666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.5266666666666666, "Vietnamese,Chinese,Malay,English": 0.47333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.5266666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.5, "Vietnamese,Spanish,Malay,English": 0.44666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.5, "Vietnamese,Malay,Indonesian,English": 0.5266666666666666, "Chinese,Spanish,Malay,Indonesian": 0.46, "Chinese,Spanish,Malay,English": 0.43333333333333335, "Chinese,Spanish,Indonesian,English": 0.48, "Chinese,Malay,Indonesian,English": 0.4866666666666667, "Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.42, "Filipino,Vietnamese,Chinese,Spanish,English": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.43333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.43333333333333335, "Filipino,Vietnamese,Spanish,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.43333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.46, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.4, "Filipino,Chinese,Spanish,Malay,English": 0.3933333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.41333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.42, "Filipino,Spanish,Malay,Indonesian,English": 0.42, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.42, "Vietnamese,Chinese,Spanish,Malay,English": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4533333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.44666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4266666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.41333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.3933333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.3933333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3933333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.35333333333333333 } }, "AC3_2": 0.6947459120228495, "AC3_3": 0.619115594555984, "AC3_4": 0.5663107980303386, "AC3_5": 0.5266852772924975, "AC3_6": 0.4953034875464393, "AC3_7": 0.46919082121141453 }, "prompt_4": { "overall_acc": 0.7590476190476191, "language_acc": { "Filipino": 0.7066666666666667, "Vietnamese": 0.7533333333333333, "Chinese": 0.7666666666666667, "Spanish": 0.76, "Malay": 0.7533333333333333, "Indonesian": 0.76, "English": 0.8133333333333334 }, "consistency_score_2": 0.7530158730158728, "consistency_score_3": 0.6451428571428571, "consistency_score_4": 0.5758095238095239, "consistency_score_5": 0.5244444444444445, "consistency_score_6": 0.4847619047619048, "consistency_score_7": 0.4533333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.68, "Filipino,Chinese": 0.6866666666666666, "Filipino,Spanish": 0.7866666666666666, "Filipino,Malay": 0.7, "Filipino,Indonesian": 0.6866666666666666, "Filipino,English": 0.76, "Vietnamese,Chinese": 0.7733333333333333, "Vietnamese,Spanish": 0.78, "Vietnamese,Malay": 0.7466666666666667, "Vietnamese,Indonesian": 0.7533333333333333, "Vietnamese,English": 0.78, "Chinese,Spanish": 0.7933333333333333, "Chinese,Malay": 0.7333333333333333, "Chinese,Indonesian": 0.7466666666666667, "Chinese,English": 0.7933333333333333, "Spanish,Malay": 0.7466666666666667, "Spanish,Indonesian": 0.74, "Spanish,English": 0.78, "Malay,Indonesian": 0.78, "Malay,English": 0.76, "Indonesian,English": 0.8066666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5866666666666667, "Filipino,Vietnamese,Spanish": 0.64, "Filipino,Vietnamese,Malay": 0.58, "Filipino,Vietnamese,Indonesian": 0.5733333333333334, "Filipino,Vietnamese,English": 0.6266666666666667, "Filipino,Chinese,Spanish": 0.66, "Filipino,Chinese,Malay": 0.58, "Filipino,Chinese,Indonesian": 0.58, "Filipino,Chinese,English": 0.64, "Filipino,Spanish,Malay": 0.64, "Filipino,Spanish,Indonesian": 0.62, "Filipino,Spanish,English": 0.68, "Filipino,Malay,Indonesian": 0.5933333333333334, "Filipino,Malay,English": 0.6333333333333333, "Filipino,Indonesian,English": 0.6466666666666666, "Vietnamese,Chinese,Spanish": 0.68, "Vietnamese,Chinese,Malay": 0.6333333333333333, "Vietnamese,Chinese,Indonesian": 0.6466666666666666, "Vietnamese,Chinese,English": 0.68, "Vietnamese,Spanish,Malay": 0.64, "Vietnamese,Spanish,Indonesian": 0.6533333333333333, "Vietnamese,Spanish,English": 0.6866666666666666, "Vietnamese,Malay,Indonesian": 0.66, "Vietnamese,Malay,English": 0.66, "Vietnamese,Indonesian,English": 0.6933333333333334, "Chinese,Spanish,Malay": 0.6466666666666666, "Chinese,Spanish,Indonesian": 0.6533333333333333, "Chinese,Spanish,English": 0.6933333333333334, "Chinese,Malay,Indonesian": 0.6533333333333333, "Chinese,Malay,English": 0.66, "Chinese,Indonesian,English": 0.6866666666666666, "Spanish,Malay,Indonesian": 0.6466666666666666, "Spanish,Malay,English": 0.66, "Spanish,Indonesian,English": 0.6733333333333333, "Malay,Indonesian,English": 0.6933333333333334 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.5733333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.5066666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.5066666666666667, "Filipino,Vietnamese,Chinese,English": 0.5533333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.56, "Filipino,Vietnamese,Spanish,Indonesian": 0.5466666666666666, "Filipino,Vietnamese,Spanish,English": 0.5933333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.52, "Filipino,Vietnamese,Malay,English": 0.5533333333333333, "Filipino,Vietnamese,Indonesian,English": 0.56, "Filipino,Chinese,Spanish,Malay": 0.56, "Filipino,Chinese,Spanish,Indonesian": 0.56, "Filipino,Chinese,Spanish,English": 0.6133333333333333, "Filipino,Chinese,Malay,Indonesian": 0.5266666666666666, "Filipino,Chinese,Malay,English": 0.56, "Filipino,Chinese,Indonesian,English": 0.5666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.56, "Filipino,Spanish,Malay,English": 0.6, "Filipino,Spanish,Indonesian,English": 0.5933333333333334, "Filipino,Malay,Indonesian,English": 0.58, "Vietnamese,Chinese,Spanish,Malay": 0.58, "Vietnamese,Chinese,Spanish,Indonesian": 0.5933333333333334, "Vietnamese,Chinese,Spanish,English": 0.62, "Vietnamese,Chinese,Malay,Indonesian": 0.5733333333333334, "Vietnamese,Chinese,Malay,English": 0.5866666666666667, "Vietnamese,Chinese,Indonesian,English": 0.6066666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.58, "Vietnamese,Spanish,Malay,English": 0.5866666666666667, "Vietnamese,Spanish,Indonesian,English": 0.6133333333333333, "Vietnamese,Malay,Indonesian,English": 0.6133333333333333, "Chinese,Spanish,Malay,Indonesian": 0.5866666666666667, "Chinese,Spanish,Malay,English": 0.5933333333333334, "Chinese,Spanish,Indonesian,English": 0.6133333333333333, "Chinese,Malay,Indonesian,English": 0.6133333333333333, "Spanish,Malay,Indonesian,English": 0.6 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.5, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.5, "Filipino,Vietnamese,Chinese,Spanish,English": 0.54, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.46, "Filipino,Vietnamese,Chinese,Malay,English": 0.49333333333333335, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.5, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.5, "Filipino,Vietnamese,Spanish,Malay,English": 0.5333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.5133333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.5066666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.54, "Filipino,Chinese,Spanish,Indonesian,English": 0.5466666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.52, "Filipino,Spanish,Malay,Indonesian,English": 0.5466666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.5333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.54, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.56, "Vietnamese,Chinese,Malay,Indonesian,English": 0.5466666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.5466666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.5533333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4533333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.4866666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.49333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.49333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.5, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.5066666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4533333333333333 } }, "AC3_2": 0.7560197154154438, "AC3_3": 0.6974753894550185, "AC3_4": 0.6548518699226752, "AC3_5": 0.6203050539544738, "AC3_6": 0.5916619265913873, "AC3_7": 0.567645980576379 }, "prompt_5": { "overall_acc": 0.6723809523809524, "language_acc": { "Filipino": 0.6533333333333333, "Vietnamese": 0.68, "Chinese": 0.62, "Spanish": 0.6466666666666666, "Malay": 0.6933333333333334, "Indonesian": 0.7, "English": 0.7133333333333334 }, "consistency_score_2": 0.6742857142857142, "consistency_score_3": 0.5386666666666667, "consistency_score_4": 0.4605714285714285, "consistency_score_5": 0.40793650793650793, "consistency_score_6": 0.36952380952380953, "consistency_score_7": 0.34, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6666666666666666, "Filipino,Chinese": 0.6466666666666666, "Filipino,Spanish": 0.64, "Filipino,Malay": 0.62, "Filipino,Indonesian": 0.6533333333333333, "Filipino,English": 0.6666666666666666, "Vietnamese,Chinese": 0.68, "Vietnamese,Spanish": 0.6733333333333333, "Vietnamese,Malay": 0.7, "Vietnamese,Indonesian": 0.7, "Vietnamese,English": 0.68, "Chinese,Spanish": 0.6466666666666666, "Chinese,Malay": 0.6466666666666666, "Chinese,Indonesian": 0.6466666666666666, "Chinese,English": 0.6666666666666666, "Spanish,Malay": 0.6466666666666666, "Spanish,Indonesian": 0.68, "Spanish,English": 0.7333333333333333, "Malay,Indonesian": 0.7666666666666667, "Malay,English": 0.7066666666666667, "Indonesian,English": 0.6933333333333334 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5266666666666666, "Filipino,Vietnamese,Spanish": 0.5133333333333333, "Filipino,Vietnamese,Malay": 0.5133333333333333, "Filipino,Vietnamese,Indonesian": 0.5266666666666666, "Filipino,Vietnamese,English": 0.5333333333333333, "Filipino,Chinese,Spanish": 0.5, "Filipino,Chinese,Malay": 0.5, "Filipino,Chinese,Indonesian": 0.5133333333333333, "Filipino,Chinese,English": 0.52, "Filipino,Spanish,Malay": 0.4866666666666667, "Filipino,Spanish,Indonesian": 0.5133333333333333, "Filipino,Spanish,English": 0.5466666666666666, "Filipino,Malay,Indonesian": 0.5466666666666666, "Filipino,Malay,English": 0.5333333333333333, "Filipino,Indonesian,English": 0.5333333333333333, "Vietnamese,Chinese,Spanish": 0.5333333333333333, "Vietnamese,Chinese,Malay": 0.54, "Vietnamese,Chinese,Indonesian": 0.54, "Vietnamese,Chinese,English": 0.5533333333333333, "Vietnamese,Spanish,Malay": 0.5266666666666666, "Vietnamese,Spanish,Indonesian": 0.5466666666666666, "Vietnamese,Spanish,English": 0.5666666666666667, "Vietnamese,Malay,Indonesian": 0.6066666666666667, "Vietnamese,Malay,English": 0.5666666666666667, "Vietnamese,Indonesian,English": 0.56, "Chinese,Spanish,Malay": 0.5133333333333333, "Chinese,Spanish,Indonesian": 0.52, "Chinese,Spanish,English": 0.56, "Chinese,Malay,Indonesian": 0.5533333333333333, "Chinese,Malay,English": 0.5466666666666666, "Chinese,Indonesian,English": 0.5266666666666666, "Spanish,Malay,Indonesian": 0.5666666666666667, "Spanish,Malay,English": 0.5533333333333333, "Spanish,Indonesian,English": 0.5666666666666667, "Malay,Indonesian,English": 0.6 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Indonesian": 0.44666666666666666, "Filipino,Vietnamese,Chinese,English": 0.44666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.42, "Filipino,Vietnamese,Spanish,Indonesian": 0.43333333333333335, "Filipino,Vietnamese,Spanish,English": 0.46, "Filipino,Vietnamese,Malay,Indonesian": 0.47333333333333333, "Filipino,Vietnamese,Malay,English": 0.4533333333333333, "Filipino,Vietnamese,Indonesian,English": 0.46, "Filipino,Chinese,Spanish,Malay": 0.4266666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.4266666666666667, "Filipino,Chinese,Spanish,English": 0.4533333333333333, "Filipino,Chinese,Malay,Indonesian": 0.4666666666666667, "Filipino,Chinese,Malay,English": 0.44666666666666666, "Filipino,Chinese,Indonesian,English": 0.44, "Filipino,Spanish,Malay,Indonesian": 0.4533333333333333, "Filipino,Spanish,Malay,English": 0.4533333333333333, "Filipino,Spanish,Indonesian,English": 0.4533333333333333, "Filipino,Malay,Indonesian,English": 0.48, "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.44666666666666666, "Vietnamese,Chinese,Spanish,English": 0.48, "Vietnamese,Chinese,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Chinese,Malay,English": 0.47333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.4666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Spanish,Malay,English": 0.48, "Vietnamese,Spanish,Indonesian,English": 0.49333333333333335, "Vietnamese,Malay,Indonesian,English": 0.5, "Chinese,Spanish,Malay,Indonesian": 0.47333333333333333, "Chinese,Spanish,Malay,English": 0.47333333333333333, "Chinese,Spanish,Indonesian,English": 0.47333333333333333, "Chinese,Malay,Indonesian,English": 0.48, "Spanish,Malay,Indonesian,English": 0.5 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Spanish,English": 0.3933333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.3933333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4, "Filipino,Vietnamese,Spanish,Malay,English": 0.4, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.4066666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.42, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.4066666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.4, "Filipino,Chinese,Spanish,Indonesian,English": 0.3933333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.41333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.42, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.42, "Vietnamese,Chinese,Spanish,Malay,English": 0.42, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.42, "Vietnamese,Chinese,Malay,Indonesian,English": 0.4266666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.44666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.44 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.38, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.38, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.34 } }, "AC3_2": 0.6733319862098505, "AC3_3": 0.5981419733174965, "AC3_4": 0.5466769141606053, "AC3_5": 0.5077928438022606, "AC3_6": 0.4769356663597265, "AC3_7": 0.451627469381542 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.6347402597402597, "language_acc": { "Indonesian": 0.6079545454545454, "English": 0.7329545454545454, "Filipino": 0.5511363636363636, "Spanish": 0.6420454545454546, "Chinese": 0.6704545454545454, "Malay": 0.5965909090909091, "Vietnamese": 0.6420454545454546 }, "consistency_score_2": 0.702651515151515, "consistency_score_3": 0.5780844155844157, "consistency_score_4": 0.504383116883117, "consistency_score_5": 0.4540043290043291, "consistency_score_6": 0.4180194805194806, "consistency_score_7": 0.39204545454545453, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6931818181818182, "Indonesian,Filipino": 0.6931818181818182, "Indonesian,Spanish": 0.7613636363636364, "Indonesian,Chinese": 0.6534090909090909, "Indonesian,Malay": 0.7897727272727273, "Indonesian,Vietnamese": 0.7443181818181818, "English,Filipino": 0.6306818181818182, "English,Spanish": 0.7386363636363636, "English,Chinese": 0.7159090909090909, "English,Malay": 0.6875, "English,Vietnamese": 0.7159090909090909, "Filipino,Spanish": 0.6761363636363636, "Filipino,Chinese": 0.6079545454545454, "Filipino,Malay": 0.7329545454545454, "Filipino,Vietnamese": 0.6534090909090909, "Spanish,Chinese": 0.6875, "Spanish,Malay": 0.7670454545454546, "Spanish,Vietnamese": 0.7556818181818182, "Chinese,Malay": 0.6590909090909091, "Chinese,Vietnamese": 0.6704545454545454, "Malay,Vietnamese": 0.7215909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.5340909090909091, "Indonesian,English,Spanish": 0.6193181818181818, "Indonesian,English,Chinese": 0.5625, "Indonesian,English,Malay": 0.5965909090909091, "Indonesian,English,Vietnamese": 0.6079545454545454, "Indonesian,Filipino,Spanish": 0.5852272727272727, "Indonesian,Filipino,Chinese": 0.5056818181818182, "Indonesian,Filipino,Malay": 0.625, "Indonesian,Filipino,Vietnamese": 0.5681818181818182, "Indonesian,Spanish,Chinese": 0.5681818181818182, "Indonesian,Spanish,Malay": 0.6818181818181818, "Indonesian,Spanish,Vietnamese": 0.6534090909090909, "Indonesian,Chinese,Malay": 0.5738636363636364, "Indonesian,Chinese,Vietnamese": 0.5568181818181818, "Indonesian,Malay,Vietnamese": 0.6363636363636364, "English,Filipino,Spanish": 0.5454545454545454, "English,Filipino,Chinese": 0.5113636363636364, "English,Filipino,Malay": 0.5511363636363636, "English,Filipino,Vietnamese": 0.5284090909090909, "English,Spanish,Chinese": 0.5852272727272727, "English,Spanish,Malay": 0.6193181818181818, "English,Spanish,Vietnamese": 0.6306818181818182, "English,Chinese,Malay": 0.5568181818181818, "English,Chinese,Vietnamese": 0.5738636363636364, "English,Malay,Vietnamese": 0.5852272727272727, "Filipino,Spanish,Chinese": 0.5227272727272727, "Filipino,Spanish,Malay": 0.6079545454545454, "Filipino,Spanish,Vietnamese": 0.5738636363636364, "Filipino,Chinese,Malay": 0.5340909090909091, "Filipino,Chinese,Vietnamese": 0.4943181818181818, "Filipino,Malay,Vietnamese": 0.5795454545454546, "Spanish,Chinese,Malay": 0.5852272727272727, "Spanish,Chinese,Vietnamese": 0.5852272727272727, "Spanish,Malay,Vietnamese": 0.6420454545454546, "Chinese,Malay,Vietnamese": 0.5454545454545454 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.5, "Indonesian,English,Filipino,Chinese": 0.4602272727272727, "Indonesian,English,Filipino,Malay": 0.4943181818181818, "Indonesian,English,Filipino,Vietnamese": 0.48295454545454547, "Indonesian,English,Spanish,Chinese": 0.5, "Indonesian,English,Spanish,Malay": 0.5625, "Indonesian,English,Spanish,Vietnamese": 0.5625, "Indonesian,English,Chinese,Malay": 0.5056818181818182, "Indonesian,English,Chinese,Vietnamese": 0.5, "Indonesian,English,Malay,Vietnamese": 0.5284090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.4659090909090909, "Indonesian,Filipino,Spanish,Malay": 0.5568181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.5284090909090909, "Indonesian,Filipino,Chinese,Malay": 0.48295454545454547, "Indonesian,Filipino,Chinese,Vietnamese": 0.44886363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.5284090909090909, "Indonesian,Spanish,Chinese,Malay": 0.5284090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.5113636363636364, "Indonesian,Spanish,Malay,Vietnamese": 0.5965909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.5, "English,Filipino,Spanish,Chinese": 0.4659090909090909, "English,Filipino,Spanish,Malay": 0.5056818181818182, "English,Filipino,Spanish,Vietnamese": 0.4943181818181818, "English,Filipino,Chinese,Malay": 0.4715909090909091, "English,Filipino,Chinese,Vietnamese": 0.45454545454545453, "English,Filipino,Malay,Vietnamese": 0.48863636363636365, "English,Spanish,Chinese,Malay": 0.5113636363636364, "English,Spanish,Chinese,Vietnamese": 0.5227272727272727, "English,Spanish,Malay,Vietnamese": 0.5454545454545454, "English,Chinese,Malay,Vietnamese": 0.4943181818181818, "Filipino,Spanish,Chinese,Malay": 0.48295454545454547, "Filipino,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.5340909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.4602272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.5113636363636364 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4318181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.4715909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.4659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.4375, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.42045454545454547, "Indonesian,English,Filipino,Malay,Vietnamese": 0.44886363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.4715909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5113636363636364, "Indonesian,English,Chinese,Malay,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.44886363636363635, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4318181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.5056818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4772727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.4375, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4318181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.4659090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "English,Spanish,Chinese,Malay,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4375 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.4147727272727273, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3977272727272727, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4375, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4147727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.39204545454545453 } }, "AC3_2": 0.6669716586778062, "AC3_3": 0.6050890281752638, "AC3_4": 0.5621028892232748, "AC3_5": 0.5293708527566243, "AC3_6": 0.5040728353665118, "AC3_7": 0.4847107437544463 }, "prompt_2": { "overall_acc": 0.6525974025974026, "language_acc": { "Indonesian": 0.6306818181818182, "English": 0.7386363636363636, "Filipino": 0.5511363636363636, "Spanish": 0.6534090909090909, "Chinese": 0.6818181818181818, "Malay": 0.6420454545454546, "Vietnamese": 0.6704545454545454 }, "consistency_score_2": 0.6915584415584416, "consistency_score_3": 0.5625, "consistency_score_4": 0.4834415584415585, "consistency_score_5": 0.4272186147186148, "consistency_score_6": 0.3847402597402597, "consistency_score_7": 0.3522727272727273, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7102272727272727, "Indonesian,Filipino": 0.6704545454545454, "Indonesian,Spanish": 0.7556818181818182, "Indonesian,Chinese": 0.6363636363636364, "Indonesian,Malay": 0.7954545454545454, "Indonesian,Vietnamese": 0.7784090909090909, "English,Filipino": 0.5965909090909091, "English,Spanish": 0.7215909090909091, "English,Chinese": 0.6534090909090909, "English,Malay": 0.7329545454545454, "English,Vietnamese": 0.7272727272727273, "Filipino,Spanish": 0.6647727272727273, "Filipino,Chinese": 0.5681818181818182, "Filipino,Malay": 0.7329545454545454, "Filipino,Vietnamese": 0.6306818181818182, "Spanish,Chinese": 0.6534090909090909, "Spanish,Malay": 0.7443181818181818, "Spanish,Vietnamese": 0.7613636363636364, "Chinese,Malay": 0.6193181818181818, "Chinese,Vietnamese": 0.6590909090909091, "Malay,Vietnamese": 0.7102272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.5227272727272727, "Indonesian,English,Spanish": 0.6193181818181818, "Indonesian,English,Chinese": 0.5227272727272727, "Indonesian,English,Malay": 0.6306818181818182, "Indonesian,English,Vietnamese": 0.6193181818181818, "Indonesian,Filipino,Spanish": 0.5681818181818182, "Indonesian,Filipino,Chinese": 0.48863636363636365, "Indonesian,Filipino,Malay": 0.625, "Indonesian,Filipino,Vietnamese": 0.5625, "Indonesian,Spanish,Chinese": 0.5454545454545454, "Indonesian,Spanish,Malay": 0.6590909090909091, "Indonesian,Spanish,Vietnamese": 0.6590909090909091, "Indonesian,Chinese,Malay": 0.5511363636363636, "Indonesian,Chinese,Vietnamese": 0.5511363636363636, "Indonesian,Malay,Vietnamese": 0.6477272727272727, "English,Filipino,Spanish": 0.5284090909090909, "English,Filipino,Chinese": 0.45454545454545453, "English,Filipino,Malay": 0.5568181818181818, "English,Filipino,Vietnamese": 0.5056818181818182, "English,Spanish,Chinese": 0.5340909090909091, "English,Spanish,Malay": 0.625, "English,Spanish,Vietnamese": 0.6306818181818182, "English,Chinese,Malay": 0.5340909090909091, "English,Chinese,Vietnamese": 0.5340909090909091, "English,Malay,Vietnamese": 0.6022727272727273, "Filipino,Spanish,Chinese": 0.5, "Filipino,Spanish,Malay": 0.5909090909090909, "Filipino,Spanish,Vietnamese": 0.5454545454545454, "Filipino,Chinese,Malay": 0.5056818181818182, "Filipino,Chinese,Vietnamese": 0.4602272727272727, "Filipino,Malay,Vietnamese": 0.5625, "Spanish,Chinese,Malay": 0.5340909090909091, "Spanish,Chinese,Vietnamese": 0.5625, "Spanish,Malay,Vietnamese": 0.625, "Chinese,Malay,Vietnamese": 0.5227272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.48863636363636365, "Indonesian,English,Filipino,Chinese": 0.42045454545454547, "Indonesian,English,Filipino,Malay": 0.5, "Indonesian,English,Filipino,Vietnamese": 0.4602272727272727, "Indonesian,English,Spanish,Chinese": 0.48295454545454547, "Indonesian,English,Spanish,Malay": 0.5625, "Indonesian,English,Spanish,Vietnamese": 0.5625, "Indonesian,English,Chinese,Malay": 0.48295454545454547, "Indonesian,English,Chinese,Vietnamese": 0.4715909090909091, "Indonesian,English,Malay,Vietnamese": 0.5511363636363636, "Indonesian,Filipino,Spanish,Chinese": 0.44886363636363635, "Indonesian,Filipino,Spanish,Malay": 0.5397727272727273, "Indonesian,Filipino,Spanish,Vietnamese": 0.5056818181818182, "Indonesian,Filipino,Chinese,Malay": 0.4659090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Chinese,Malay": 0.4943181818181818, "Indonesian,Spanish,Chinese,Vietnamese": 0.48295454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.5738636363636364, "Indonesian,Chinese,Malay,Vietnamese": 0.4772727272727273, "English,Filipino,Spanish,Chinese": 0.42613636363636365, "English,Filipino,Spanish,Malay": 0.5113636363636364, "English,Filipino,Spanish,Vietnamese": 0.4659090909090909, "English,Filipino,Chinese,Malay": 0.4431818181818182, "English,Filipino,Chinese,Vietnamese": 0.3977272727272727, "English,Filipino,Malay,Vietnamese": 0.4772727272727273, "English,Spanish,Chinese,Malay": 0.4772727272727273, "English,Spanish,Chinese,Vietnamese": 0.48863636363636365, "English,Spanish,Malay,Vietnamese": 0.5454545454545454, "English,Chinese,Malay,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Chinese,Malay": 0.45454545454545453, "Filipino,Spanish,Chinese,Vietnamese": 0.4318181818181818, "Filipino,Spanish,Malay,Vietnamese": 0.5113636363636364, "Filipino,Chinese,Malay,Vietnamese": 0.4318181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.4715909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4034090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.4715909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.4375, "Indonesian,English,Filipino,Chinese,Malay": 0.4090909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.375, "Indonesian,English,Filipino,Malay,Vietnamese": 0.4431818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.44886363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4375, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5056818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4318181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.42613636363636365, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.3977272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.4034090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4318181818181818, "English,Filipino,Spanish,Chinese,Malay": 0.4147727272727273, "English,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.44886363636363635, "English,Filipino,Chinese,Malay,Vietnamese": 0.38636363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.4318181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4034090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.39204545454545453, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.36363636363636365, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.42045454545454547, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.36363636363636365, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4034090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3522727272727273 } }, "AC3_2": 0.6715132692893984, "AC3_3": 0.6042084167839422, "AC3_4": 0.5554258544624161, "AC3_5": 0.5163875212256931, "AC3_6": 0.484086336152346, "AC3_7": 0.4575561756043413 }, "prompt_3": { "overall_acc": 0.6331168831168832, "language_acc": { "Indonesian": 0.6079545454545454, "English": 0.7045454545454546, "Filipino": 0.5625, "Spanish": 0.6136363636363636, "Chinese": 0.6761363636363636, "Malay": 0.6136363636363636, "Vietnamese": 0.6534090909090909 }, "consistency_score_2": 0.6720779220779222, "consistency_score_3": 0.536525974025974, "consistency_score_4": 0.45795454545454545, "consistency_score_5": 0.40584415584415595, "consistency_score_6": 0.3685064935064935, "consistency_score_7": 0.3409090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6420454545454546, "Indonesian,Filipino": 0.6875, "Indonesian,Spanish": 0.7443181818181818, "Indonesian,Chinese": 0.6136363636363636, "Indonesian,Malay": 0.7556818181818182, "Indonesian,Vietnamese": 0.7329545454545454, "English,Filipino": 0.5795454545454546, "English,Spanish": 0.7045454545454546, "English,Chinese": 0.6818181818181818, "English,Malay": 0.6590909090909091, "English,Vietnamese": 0.7159090909090909, "Filipino,Spanish": 0.6363636363636364, "Filipino,Chinese": 0.5738636363636364, "Filipino,Malay": 0.6931818181818182, "Filipino,Vietnamese": 0.6306818181818182, "Spanish,Chinese": 0.6136363636363636, "Spanish,Malay": 0.7045454545454546, "Spanish,Vietnamese": 0.7443181818181818, "Chinese,Malay": 0.6136363636363636, "Chinese,Vietnamese": 0.6534090909090909, "Malay,Vietnamese": 0.7329545454545454 }, "3_combine": { "Indonesian,English,Filipino": 0.48863636363636365, "Indonesian,English,Spanish": 0.5738636363636364, "Indonesian,English,Chinese": 0.5056818181818182, "Indonesian,English,Malay": 0.5454545454545454, "Indonesian,English,Vietnamese": 0.5681818181818182, "Indonesian,Filipino,Spanish": 0.5568181818181818, "Indonesian,Filipino,Chinese": 0.48295454545454547, "Indonesian,Filipino,Malay": 0.5852272727272727, "Indonesian,Filipino,Vietnamese": 0.5511363636363636, "Indonesian,Spanish,Chinese": 0.5056818181818182, "Indonesian,Spanish,Malay": 0.6136363636363636, "Indonesian,Spanish,Vietnamese": 0.6306818181818182, "Indonesian,Chinese,Malay": 0.5170454545454546, "Indonesian,Chinese,Vietnamese": 0.5284090909090909, "Indonesian,Malay,Vietnamese": 0.625, "English,Filipino,Spanish": 0.5, "English,Filipino,Chinese": 0.4602272727272727, "English,Filipino,Malay": 0.5, "English,Filipino,Vietnamese": 0.48863636363636365, "English,Spanish,Chinese": 0.5227272727272727, "English,Spanish,Malay": 0.5511363636363636, "English,Spanish,Vietnamese": 0.6079545454545454, "English,Chinese,Malay": 0.5113636363636364, "English,Chinese,Vietnamese": 0.5568181818181818, "English,Malay,Vietnamese": 0.5738636363636364, "Filipino,Spanish,Chinese": 0.4715909090909091, "Filipino,Spanish,Malay": 0.5511363636363636, "Filipino,Spanish,Vietnamese": 0.5397727272727273, "Filipino,Chinese,Malay": 0.48295454545454547, "Filipino,Chinese,Vietnamese": 0.4659090909090909, "Filipino,Malay,Vietnamese": 0.5568181818181818, "Spanish,Chinese,Malay": 0.4943181818181818, "Spanish,Chinese,Vietnamese": 0.5340909090909091, "Spanish,Malay,Vietnamese": 0.6022727272727273, "Chinese,Malay,Vietnamese": 0.5284090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.4602272727272727, "Indonesian,English,Filipino,Chinese": 0.4147727272727273, "Indonesian,English,Filipino,Malay": 0.44886363636363635, "Indonesian,English,Filipino,Vietnamese": 0.44886363636363635, "Indonesian,English,Spanish,Chinese": 0.45454545454545453, "Indonesian,English,Spanish,Malay": 0.4943181818181818, "Indonesian,English,Spanish,Vietnamese": 0.5284090909090909, "Indonesian,English,Chinese,Malay": 0.4431818181818182, "Indonesian,English,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,English,Malay,Vietnamese": 0.5, "Indonesian,Filipino,Spanish,Chinese": 0.42045454545454547, "Indonesian,Filipino,Spanish,Malay": 0.5, "Indonesian,Filipino,Spanish,Vietnamese": 0.5, "Indonesian,Filipino,Chinese,Malay": 0.42045454545454547, "Indonesian,Filipino,Chinese,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.5056818181818182, "Indonesian,Spanish,Chinese,Malay": 0.4375, "Indonesian,Spanish,Chinese,Vietnamese": 0.45454545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.5397727272727273, "Indonesian,Chinese,Malay,Vietnamese": 0.4602272727272727, "English,Filipino,Spanish,Chinese": 0.4090909090909091, "English,Filipino,Spanish,Malay": 0.45454545454545453, "English,Filipino,Spanish,Vietnamese": 0.45454545454545453, "English,Filipino,Chinese,Malay": 0.42045454545454547, "English,Filipino,Chinese,Vietnamese": 0.4090909090909091, "English,Filipino,Malay,Vietnamese": 0.45454545454545453, "English,Spanish,Chinese,Malay": 0.4318181818181818, "English,Spanish,Chinese,Vietnamese": 0.48295454545454547, "English,Spanish,Malay,Vietnamese": 0.5, "English,Chinese,Malay,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Chinese,Malay": 0.42613636363636365, "Filipino,Spanish,Chinese,Vietnamese": 0.42613636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.5056818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.44886363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.38636363636363635, "Indonesian,English,Filipino,Spanish,Malay": 0.42613636363636365, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.4318181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.3806818181818182, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.3806818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.42045454545454547, "Indonesian,English,Spanish,Chinese,Malay": 0.3977272727272727, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.42613636363636365, "Indonesian,English,Spanish,Malay,Vietnamese": 0.4602272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3806818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.3806818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.3806818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.42613636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.38636363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.35795454545454547, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.36363636363636365, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4034090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.35795454545454547, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.375, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.35795454545454547, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.36363636363636365 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3409090909090909 } }, "AC3_2": 0.6520158945032527, "AC3_3": 0.5808331154815828, "AC3_4": 0.531475294482676, "AC3_5": 0.49462256488745765, "AC3_6": 0.46585910033447003, "AC3_7": 0.4431818181363182 }, "prompt_4": { "overall_acc": 0.6452922077922079, "language_acc": { "Indonesian": 0.6363636363636364, "English": 0.7443181818181818, "Filipino": 0.5340909090909091, "Spanish": 0.625, "Chinese": 0.6931818181818182, "Malay": 0.6136363636363636, "Vietnamese": 0.6704545454545454 }, "consistency_score_2": 0.7056277056277056, "consistency_score_3": 0.5840909090909091, "consistency_score_4": 0.5105519480519481, "consistency_score_5": 0.4591450216450217, "consistency_score_6": 0.42126623376623373, "consistency_score_7": 0.39204545454545453, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7102272727272727, "Indonesian,Filipino": 0.6875, "Indonesian,Spanish": 0.7613636363636364, "Indonesian,Chinese": 0.6477272727272727, "Indonesian,Malay": 0.8238636363636364, "Indonesian,Vietnamese": 0.7897727272727273, "English,Filipino": 0.6136363636363636, "English,Spanish": 0.7386363636363636, "English,Chinese": 0.7102272727272727, "English,Malay": 0.7102272727272727, "English,Vietnamese": 0.7159090909090909, "Filipino,Spanish": 0.6761363636363636, "Filipino,Chinese": 0.5909090909090909, "Filipino,Malay": 0.7556818181818182, "Filipino,Vietnamese": 0.6193181818181818, "Spanish,Chinese": 0.7045454545454546, "Spanish,Malay": 0.7784090909090909, "Spanish,Vietnamese": 0.7443181818181818, "Chinese,Malay": 0.6477272727272727, "Chinese,Vietnamese": 0.6704545454545454, "Malay,Vietnamese": 0.7215909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.5340909090909091, "Indonesian,English,Spanish": 0.6193181818181818, "Indonesian,English,Chinese": 0.5568181818181818, "Indonesian,English,Malay": 0.6363636363636364, "Indonesian,English,Vietnamese": 0.6363636363636364, "Indonesian,Filipino,Spanish": 0.5909090909090909, "Indonesian,Filipino,Chinese": 0.5056818181818182, "Indonesian,Filipino,Malay": 0.6534090909090909, "Indonesian,Filipino,Vietnamese": 0.5738636363636364, "Indonesian,Spanish,Chinese": 0.5738636363636364, "Indonesian,Spanish,Malay": 0.6988636363636364, "Indonesian,Spanish,Vietnamese": 0.6647727272727273, "Indonesian,Chinese,Malay": 0.5852272727272727, "Indonesian,Chinese,Vietnamese": 0.5738636363636364, "Indonesian,Malay,Vietnamese": 0.6761363636363636, "English,Filipino,Spanish": 0.5397727272727273, "English,Filipino,Chinese": 0.48863636363636365, "English,Filipino,Malay": 0.5681818181818182, "English,Filipino,Vietnamese": 0.5056818181818182, "English,Spanish,Chinese": 0.5909090909090909, "English,Spanish,Malay": 0.6420454545454546, "English,Spanish,Vietnamese": 0.6136363636363636, "English,Chinese,Malay": 0.5681818181818182, "English,Chinese,Vietnamese": 0.5681818181818182, "English,Malay,Vietnamese": 0.5965909090909091, "Filipino,Spanish,Chinese": 0.5340909090909091, "Filipino,Spanish,Malay": 0.6306818181818182, "Filipino,Spanish,Vietnamese": 0.5568181818181818, "Filipino,Chinese,Malay": 0.5284090909090909, "Filipino,Chinese,Vietnamese": 0.4772727272727273, "Filipino,Malay,Vietnamese": 0.5738636363636364, "Spanish,Chinese,Malay": 0.6079545454545454, "Spanish,Chinese,Vietnamese": 0.5852272727272727, "Spanish,Malay,Vietnamese": 0.6420454545454546, "Chinese,Malay,Vietnamese": 0.5454545454545454 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.4943181818181818, "Indonesian,English,Filipino,Chinese": 0.44886363636363635, "Indonesian,English,Filipino,Malay": 0.5170454545454546, "Indonesian,English,Filipino,Vietnamese": 0.4772727272727273, "Indonesian,English,Spanish,Chinese": 0.5, "Indonesian,English,Spanish,Malay": 0.5852272727272727, "Indonesian,English,Spanish,Vietnamese": 0.5568181818181818, "Indonesian,English,Chinese,Malay": 0.5227272727272727, "Indonesian,English,Chinese,Vietnamese": 0.5113636363636364, "Indonesian,English,Malay,Vietnamese": 0.5681818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.48295454545454547, "Indonesian,Filipino,Spanish,Malay": 0.5738636363636364, "Indonesian,Filipino,Spanish,Vietnamese": 0.5227272727272727, "Indonesian,Filipino,Chinese,Malay": 0.48295454545454547, "Indonesian,Filipino,Chinese,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Malay,Vietnamese": 0.5397727272727273, "Indonesian,Spanish,Chinese,Malay": 0.5511363636363636, "Indonesian,Spanish,Chinese,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.6136363636363636, "Indonesian,Chinese,Malay,Vietnamese": 0.5227272727272727, "English,Filipino,Spanish,Chinese": 0.4602272727272727, "English,Filipino,Spanish,Malay": 0.5284090909090909, "English,Filipino,Spanish,Vietnamese": 0.4715909090909091, "English,Filipino,Chinese,Malay": 0.4715909090909091, "English,Filipino,Chinese,Vietnamese": 0.42613636363636365, "English,Filipino,Malay,Vietnamese": 0.48863636363636365, "English,Spanish,Chinese,Malay": 0.5284090909090909, "English,Spanish,Chinese,Vietnamese": 0.5170454545454546, "English,Spanish,Malay,Vietnamese": 0.5511363636363636, "English,Chinese,Malay,Vietnamese": 0.4943181818181818, "Filipino,Spanish,Chinese,Malay": 0.5056818181818182, "Filipino,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.5340909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.45454545454545453, "Spanish,Chinese,Malay,Vietnamese": 0.5227272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.42613636363636365, "Indonesian,English,Filipino,Spanish,Malay": 0.48863636363636365, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.44886363636363635, "Indonesian,English,Filipino,Chinese,Malay": 0.4375, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.4602272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.48863636363636365, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5284090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4659090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.5056818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.4318181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.5, "English,Filipino,Spanish,Chinese,Malay": 0.44886363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4147727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.4659090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.4147727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.4715909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.44886363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.3977272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3977272727272727, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.39204545454545453 } }, "AC3_2": 0.6741125887432413, "AC3_3": 0.613168193172457, "AC3_4": 0.5700685374525036, "AC3_5": 0.5365315416581213, "AC3_6": 0.5097513787104341, "AC3_7": 0.4877560818992028 }, "prompt_5": { "overall_acc": 0.6217532467532468, "language_acc": { "Indonesian": 0.6136363636363636, "English": 0.6931818181818182, "Filipino": 0.5397727272727273, "Spanish": 0.625, "Chinese": 0.6363636363636364, "Malay": 0.5965909090909091, "Vietnamese": 0.6477272727272727 }, "consistency_score_2": 0.6845238095238096, "consistency_score_3": 0.5553571428571428, "consistency_score_4": 0.4806818181818182, "consistency_score_5": 0.43073593073593064, "consistency_score_6": 0.3952922077922078, "consistency_score_7": 0.3693181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6704545454545454, "Indonesian,Filipino": 0.6704545454545454, "Indonesian,Spanish": 0.7670454545454546, "Indonesian,Chinese": 0.6420454545454546, "Indonesian,Malay": 0.7613636363636364, "Indonesian,Vietnamese": 0.75, "English,Filipino": 0.6136363636363636, "English,Spanish": 0.6875, "English,Chinese": 0.6306818181818182, "English,Malay": 0.6988636363636364, "English,Vietnamese": 0.7045454545454546, "Filipino,Spanish": 0.6647727272727273, "Filipino,Chinese": 0.5340909090909091, "Filipino,Malay": 0.6988636363636364, "Filipino,Vietnamese": 0.6363636363636364, "Spanish,Chinese": 0.6818181818181818, "Spanish,Malay": 0.7727272727272727, "Spanish,Vietnamese": 0.7386363636363636, "Chinese,Malay": 0.6534090909090909, "Chinese,Vietnamese": 0.6534090909090909, "Malay,Vietnamese": 0.7443181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.5227272727272727, "Indonesian,English,Spanish": 0.5852272727272727, "Indonesian,English,Chinese": 0.5, "Indonesian,English,Malay": 0.5852272727272727, "Indonesian,English,Vietnamese": 0.5909090909090909, "Indonesian,Filipino,Spanish": 0.5795454545454546, "Indonesian,Filipino,Chinese": 0.4659090909090909, "Indonesian,Filipino,Malay": 0.5795454545454546, "Indonesian,Filipino,Vietnamese": 0.5568181818181818, "Indonesian,Spanish,Chinese": 0.5568181818181818, "Indonesian,Spanish,Malay": 0.6704545454545454, "Indonesian,Spanish,Vietnamese": 0.6420454545454546, "Indonesian,Chinese,Malay": 0.5568181818181818, "Indonesian,Chinese,Vietnamese": 0.5397727272727273, "Indonesian,Malay,Vietnamese": 0.6477272727272727, "English,Filipino,Spanish": 0.5340909090909091, "English,Filipino,Chinese": 0.44886363636363635, "English,Filipino,Malay": 0.5454545454545454, "English,Filipino,Vietnamese": 0.5227272727272727, "English,Spanish,Chinese": 0.5227272727272727, "English,Spanish,Malay": 0.6022727272727273, "English,Spanish,Vietnamese": 0.5965909090909091, "English,Chinese,Malay": 0.5227272727272727, "English,Chinese,Vietnamese": 0.5113636363636364, "English,Malay,Vietnamese": 0.5965909090909091, "Filipino,Spanish,Chinese": 0.48863636363636365, "Filipino,Spanish,Malay": 0.5909090909090909, "Filipino,Spanish,Vietnamese": 0.5568181818181818, "Filipino,Chinese,Malay": 0.48295454545454547, "Filipino,Chinese,Vietnamese": 0.44886363636363635, "Filipino,Malay,Vietnamese": 0.5681818181818182, "Spanish,Chinese,Malay": 0.5795454545454546, "Spanish,Chinese,Vietnamese": 0.5568181818181818, "Spanish,Malay,Vietnamese": 0.6420454545454546, "Chinese,Malay,Vietnamese": 0.5397727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.48863636363636365, "Indonesian,English,Filipino,Chinese": 0.42045454545454547, "Indonesian,English,Filipino,Malay": 0.48863636363636365, "Indonesian,English,Filipino,Vietnamese": 0.4772727272727273, "Indonesian,English,Spanish,Chinese": 0.45454545454545453, "Indonesian,English,Spanish,Malay": 0.5397727272727273, "Indonesian,English,Spanish,Vietnamese": 0.5454545454545454, "Indonesian,English,Chinese,Malay": 0.4659090909090909, "Indonesian,English,Chinese,Vietnamese": 0.4602272727272727, "Indonesian,English,Malay,Vietnamese": 0.5397727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.44886363636363635, "Indonesian,Filipino,Spanish,Malay": 0.5340909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.5056818181818182, "Indonesian,Filipino,Chinese,Malay": 0.4431818181818182, "Indonesian,Filipino,Chinese,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.5056818181818182, "Indonesian,Spanish,Chinese,Malay": 0.5113636363636364, "Indonesian,Spanish,Chinese,Vietnamese": 0.48295454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.5852272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Spanish,Chinese": 0.42613636363636365, "English,Filipino,Spanish,Malay": 0.5, "English,Filipino,Spanish,Vietnamese": 0.48295454545454547, "English,Filipino,Chinese,Malay": 0.42045454545454547, "English,Filipino,Chinese,Vietnamese": 0.4090909090909091, "English,Filipino,Malay,Vietnamese": 0.48863636363636365, "English,Spanish,Chinese,Malay": 0.4715909090909091, "English,Spanish,Chinese,Vietnamese": 0.4715909090909091, "English,Spanish,Malay,Vietnamese": 0.5454545454545454, "English,Chinese,Malay,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Chinese,Malay": 0.4659090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.4375, "Filipino,Spanish,Malay,Vietnamese": 0.5170454545454546, "Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, "Spanish,Chinese,Malay,Vietnamese": 0.5 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4034090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.4659090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.45454545454545453, "Indonesian,English,Filipino,Chinese,Malay": 0.4034090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.39204545454545453, "Indonesian,English,Filipino,Malay,Vietnamese": 0.44886363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.4318181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4318181818181818, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5113636363636364, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4318181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4318181818181818, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.39204545454545453, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.44886363636363635, "English,Filipino,Spanish,Chinese,Malay": 0.4090909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.3977272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.4602272727272727, "English,Filipino,Chinese,Malay,Vietnamese": 0.39204545454545453, "English,Spanish,Chinese,Malay,Vietnamese": 0.4375, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.39204545454545453, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.3806818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4375, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.375, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818 } }, "AC3_2": 0.651630370408158, "AC3_3": 0.5866826250068015, "AC3_4": 0.5421915368780352, "AC3_5": 0.5089106266000627, "AC3_6": 0.4833101853775887, "AC3_7": 0.46338694061291186 } }, "sg_eval": { "prompt_1": { "accuracy": 0.7475728155339806 }, "prompt_2": { "accuracy": 0.7669902912621359 }, "prompt_3": { "accuracy": 0.7378640776699029 }, "prompt_4": { "accuracy": 0.7184466019417476 }, "prompt_5": { "accuracy": 0.7087378640776699 } }, "cn_eval": { "prompt_1": { "accuracy": 0.5333333333333333 }, "prompt_2": { "accuracy": 0.49523809523809526 }, "prompt_3": { "accuracy": 0.49523809523809526 }, "prompt_4": { "accuracy": 0.5047619047619047 }, "prompt_5": { "accuracy": 0.44761904761904764 } }, "us_eval": { "prompt_1": { "accuracy": 0.8130841121495327 }, "prompt_2": { "accuracy": 0.822429906542056 }, "prompt_3": { "accuracy": 0.8504672897196262 }, "prompt_4": { "accuracy": 0.8411214953271028 }, "prompt_5": { "accuracy": 0.8037383177570093 } }, "ph_eval": { "prompt_1": { "accuracy": 0.67, "category_acc": { "brand": 0.7, "demographics": 0.2, "biology": 0.8, "history": 0.6, "literature": 0.4, "politics": 0.8, "culture": 0.9, "film": 0.6, "law": 0.8, "geography": 0.7 } }, "prompt_2": { "accuracy": 0.71, "category_acc": { "brand": 0.8, "demographics": 0.0, "biology": 0.8, "history": 0.7333333333333333, "literature": 0.3, "politics": 0.9, "culture": 1.0, "film": 0.6, "law": 0.8, "geography": 0.8 } }, "prompt_3": { "accuracy": 0.64, "category_acc": { "brand": 0.6, "demographics": 0.2, "biology": 0.8, "history": 0.4666666666666667, "literature": 0.3, "politics": 1.0, "culture": 1.0, "film": 0.6, "law": 0.8, "geography": 0.5 } }, "prompt_4": { "accuracy": 0.72, "category_acc": { "brand": 0.7, "demographics": 0.2, "biology": 0.8, "history": 0.7333333333333333, "literature": 0.4, "politics": 0.9, "culture": 1.0, "film": 0.6, "law": 0.8, "geography": 0.8 } }, "prompt_5": { "accuracy": 0.68, "category_acc": { "brand": 0.5, "demographics": 0.4, "biology": 0.8, "history": 0.6, "literature": 0.5, "politics": 0.9, "culture": 1.0, "film": 0.6, "law": 0.8, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.06254107307284763 }, "prompt_2": { "bleu_score": 0.0637524633806702 }, "prompt_3": { "bleu_score": 0.06746019171954343 }, "prompt_4": { "bleu_score": 0.06800749115984081 }, "prompt_5": { "bleu_score": 0.056630114509915605 } }, "indommlu": { "prompt_1": { "accuracy": 0.6345550437278857, "category_acc": { "History": 0.6746987951807228, "Geography": 0.6530612244897959, "Lampungic": 0.3945578231292517, "Social science": 0.8647746243739566, "Balinese": 0.3970276008492569, "Makassarese": 0.46774193548387094, "Banjarese": 0.5416666666666666, "Chemistry": 0.3795620437956204, "Biology": 0.6792899408284023, "Science": 0.8111455108359134, "Christian religion": 0.7661691542288557, "Art": 0.7487520798668885, "Islam religion": 0.7880512091038406, "Hindu religion": 0.6933333333333334, "Madurese": 0.33220338983050846, "Sport": 0.6216216216216216, "Indonesian language": 0.6768368617683687, "Physics": 0.5717171717171717, "Minangkabau culture": 0.49246231155778897, "Dayak language": 0.3669724770642202, "Sociology": 0.6350806451612904, "Economy": 0.639344262295082, "Sundanese": 0.5108038029386344, "Javanese": 0.5060483870967742, "Civic education": 0.7639484978540773 } }, "prompt_2": { "accuracy": 0.6418318979905201, "category_acc": { "History": 0.6887550200803213, "Geography": 0.6551020408163265, "Lampungic": 0.4421768707482993, "Social science": 0.8848080133555927, "Balinese": 0.3885350318471338, "Makassarese": 0.4838709677419355, "Banjarese": 0.5, "Chemistry": 0.4058394160583942, "Biology": 0.6840236686390533, "Science": 0.8090815273477813, "Christian religion": 0.8009950248756219, "Art": 0.7587354409317804, "Islam religion": 0.7766714082503556, "Hindu religion": 0.6933333333333334, "Madurese": 0.34576271186440677, "Sport": 0.6351351351351351, "Indonesian language": 0.6802615193026152, "Physics": 0.5494949494949495, "Minangkabau culture": 0.507537688442211, "Dayak language": 0.3944954128440367, "Sociology": 0.6270161290322581, "Economy": 0.6434426229508197, "Sundanese": 0.5367329299913569, "Javanese": 0.5352822580645161, "Civic education": 0.7567954220314735 } }, "prompt_3": { "accuracy": 0.6436344215234662, "category_acc": { "History": 0.6987951807228916, "Geography": 0.6489795918367347, "Lampungic": 0.3877551020408163, "Social science": 0.8697829716193656, "Balinese": 0.3673036093418259, "Makassarese": 0.489247311827957, "Banjarese": 0.4722222222222222, "Chemistry": 0.42773722627737226, "Biology": 0.6863905325443787, "Science": 0.8163054695562435, "Christian religion": 0.8059701492537313, "Art": 0.7504159733777038, "Islam religion": 0.7752489331436699, "Hindu religion": 0.6933333333333334, "Madurese": 0.3389830508474576, "Sport": 0.6283783783783784, "Indonesian language": 0.6889788293897883, "Physics": 0.5696969696969697, "Minangkabau culture": 0.49246231155778897, "Dayak language": 0.3669724770642202, "Sociology": 0.6189516129032258, "Economy": 0.6454918032786885, "Sundanese": 0.5298184961106309, "Javanese": 0.5413306451612904, "Civic education": 0.7739628040057225 } }, "prompt_4": { "accuracy": 0.6452366646638628, "category_acc": { "History": 0.7008032128514057, "Geography": 0.6653061224489796, "Lampungic": 0.4013605442176871, "Social science": 0.8731218697829716, "Balinese": 0.3609341825902335, "Makassarese": 0.4838709677419355, "Banjarese": 0.4722222222222222, "Chemistry": 0.44233576642335765, "Biology": 0.6911242603550296, "Science": 0.8214654282765738, "Christian religion": 0.7960199004975125, "Art": 0.7487520798668885, "Islam religion": 0.786628733997155, "Hindu religion": 0.68, "Madurese": 0.34915254237288135, "Sport": 0.6486486486486487, "Indonesian language": 0.6793275217932753, "Physics": 0.5818181818181818, "Minangkabau culture": 0.48743718592964824, "Dayak language": 0.3761467889908257, "Sociology": 0.6209677419354839, "Economy": 0.6618852459016393, "Sundanese": 0.54019014693172, "Javanese": 0.5352822580645161, "Civic education": 0.7696709585121603 } }, "prompt_5": { "accuracy": 0.6349556045129848, "category_acc": { "History": 0.6867469879518072, "Geography": 0.6387755102040816, "Lampungic": 0.3877551020408163, "Social science": 0.8580968280467446, "Balinese": 0.37579617834394907, "Makassarese": 0.4731182795698925, "Banjarese": 0.5, "Chemistry": 0.4029197080291971, "Biology": 0.6828402366863905, "Science": 0.8142414860681114, "Christian religion": 0.7960199004975125, "Art": 0.7387687188019967, "Islam religion": 0.7837837837837838, "Hindu religion": 0.68, "Madurese": 0.3050847457627119, "Sport": 0.6351351351351351, "Indonesian language": 0.683997509339975, "Physics": 0.5656565656565656, "Minangkabau culture": 0.48743718592964824, "Dayak language": 0.3577981651376147, "Sociology": 0.6451612903225806, "Economy": 0.6454918032786885, "Sundanese": 0.5082108902333622, "Javanese": 0.5131048387096774, "Civic education": 0.7439198855507868 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.08681388844799684 }, "prompt_2": { "bleu_score": 0.0880334958080158 }, "prompt_3": { "bleu_score": 0.08805929258706634 }, "prompt_4": { "bleu_score": 0.08842607869136156 }, "prompt_5": { "bleu_score": 0.08768127846496257 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.07138322514226816 }, "prompt_2": { "bleu_score": 0.07471863870389119 }, "prompt_3": { "bleu_score": 0.07467792001731594 }, "prompt_4": { "bleu_score": 0.0749366075388539 }, "prompt_5": { "bleu_score": 0.07514655189733327 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.05903468503840439 }, "prompt_2": { "bleu_score": 0.057043614692943063 }, "prompt_3": { "bleu_score": 0.05784040342624702 }, "prompt_4": { "bleu_score": 0.05727950339564555 }, "prompt_5": { "bleu_score": 0.057461566054743354 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.08624501669818817 }, "prompt_2": { "bleu_score": 0.09009103541257128 }, "prompt_3": { "bleu_score": 0.09153674138343326 }, "prompt_4": { "bleu_score": 0.0910349620847283 }, "prompt_5": { "bleu_score": 0.08823269277227647 } }, "mmlu": { "prompt_1": { "accuracy": 0.7537922987164527 }, "prompt_2": { "accuracy": 0.7829638273045507 }, "prompt_3": { "accuracy": 0.7409568261376897 }, "prompt_4": { "accuracy": 0.7409568261376897 }, "prompt_5": { "accuracy": 0.6896149358226371 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.7598855917053986, "category_acc": { "high_school_european_history": 0.774390243902439, "business_ethics": 0.8080808080808081, "clinical_knowledge": 0.7765151515151515, "medical_genetics": 0.8686868686868687, "high_school_us_history": 0.8620689655172413, "high_school_physics": 0.5933333333333334, "high_school_world_history": 0.8940677966101694, "virology": 0.5515151515151515, "high_school_microeconomics": 0.8776371308016878, "econometrics": 0.6106194690265486, "college_computer_science": 0.6161616161616161, "high_school_biology": 0.8543689320388349, "abstract_algebra": 0.42424242424242425, "professional_accounting": 0.6156583629893239, "philosophy": 0.8064516129032258, "professional_medicine": 0.8708487084870848, "nutrition": 0.8327868852459016, "global_facts": 0.6161616161616161, "machine_learning": 0.5855855855855856, "security_studies": 0.7172131147540983, "public_relations": 0.7247706422018348, "professional_psychology": 0.8265139116202946, "prehistory": 0.8544891640866873, "anatomy": 0.7611940298507462, "human_sexuality": 0.8538461538461538, "college_medicine": 0.7267441860465116, "high_school_government_and_politics": 0.9479166666666666, "college_chemistry": 0.5353535353535354, "logical_fallacies": 0.8395061728395061, "high_school_geography": 0.8984771573604061, "elementary_mathematics": 0.8275862068965517, "human_aging": 0.7972972972972973, "college_mathematics": 0.42424242424242425, "high_school_psychology": 0.9117647058823529, "formal_logic": 0.64, "high_school_statistics": 0.6651162790697674, "international_law": 0.875, "high_school_mathematics": 0.4275092936802974, "high_school_computer_science": 0.8080808080808081, "conceptual_physics": 0.7735042735042735, "miscellaneous": 0.928388746803069, "high_school_chemistry": 0.6732673267326733, "marketing": 0.9527896995708155, "professional_law": 0.6001304631441617, "management": 0.8529411764705882, "college_physics": 0.5742574257425742, "jurisprudence": 0.8411214953271028, "world_religions": 0.8764705882352941, "sociology": 0.885, "us_foreign_policy": 0.9090909090909091, "high_school_macroeconomics": 0.7763496143958869, "computer_security": 0.7474747474747475, "moral_scenarios": 0.6174496644295302, "moral_disputes": 0.8028985507246377, "electrical_engineering": 0.75, "astronomy": 0.8741721854304636, "college_biology": 0.8951048951048951 } }, "prompt_2": { "accuracy": 0.7785484447622453, "category_acc": { "high_school_european_history": 0.774390243902439, "business_ethics": 0.8181818181818182, "clinical_knowledge": 0.803030303030303, "medical_genetics": 0.9191919191919192, "high_school_us_history": 0.9014778325123153, "high_school_physics": 0.6133333333333333, "high_school_world_history": 0.8898305084745762, "virology": 0.5333333333333333, "high_school_microeconomics": 0.8860759493670886, "econometrics": 0.6371681415929203, "college_computer_science": 0.6868686868686869, "high_school_biology": 0.8932038834951457, "abstract_algebra": 0.48484848484848486, "professional_accounting": 0.6192170818505338, "philosophy": 0.8032258064516129, "professional_medicine": 0.8929889298892989, "nutrition": 0.8459016393442623, "global_facts": 0.696969696969697, "machine_learning": 0.6216216216216216, "security_studies": 0.7622950819672131, "public_relations": 0.7064220183486238, "professional_psychology": 0.839607201309329, "prehistory": 0.8761609907120743, "anatomy": 0.7835820895522388, "human_sexuality": 0.8846153846153846, "college_medicine": 0.7616279069767442, "high_school_government_and_politics": 0.9583333333333334, "college_chemistry": 0.5959595959595959, "logical_fallacies": 0.8395061728395061, "high_school_geography": 0.9035532994923858, "elementary_mathematics": 0.8249336870026526, "human_aging": 0.8243243243243243, "college_mathematics": 0.42424242424242425, "high_school_psychology": 0.9264705882352942, "formal_logic": 0.648, "high_school_statistics": 0.6837209302325581, "international_law": 0.8916666666666667, "high_school_mathematics": 0.5315985130111525, "high_school_computer_science": 0.8686868686868687, "conceptual_physics": 0.8034188034188035, "miscellaneous": 0.9271099744245525, "high_school_chemistry": 0.693069306930693, "marketing": 0.944206008583691, "professional_law": 0.6203522504892368, "management": 0.8627450980392157, "college_physics": 0.6039603960396039, "jurisprudence": 0.8598130841121495, "world_religions": 0.8823529411764706, "sociology": 0.885, "us_foreign_policy": 0.9292929292929293, "high_school_macroeconomics": 0.8200514138817481, "computer_security": 0.7878787878787878, "moral_scenarios": 0.6319910514541387, "moral_disputes": 0.7942028985507247, "electrical_engineering": 0.7569444444444444, "astronomy": 0.9205298013245033, "college_biology": 0.9090909090909091 } }, "prompt_3": { "accuracy": 0.7543081873435824, "category_acc": { "high_school_european_history": 0.7804878048780488, "business_ethics": 0.8080808080808081, "clinical_knowledge": 0.7575757575757576, "medical_genetics": 0.8686868686868687, "high_school_us_history": 0.8620689655172413, "high_school_physics": 0.5933333333333334, "high_school_world_history": 0.8813559322033898, "virology": 0.5454545454545454, "high_school_microeconomics": 0.8523206751054853, "econometrics": 0.6106194690265486, "college_computer_science": 0.6262626262626263, "high_school_biology": 0.8349514563106796, "abstract_algebra": 0.45454545454545453, "professional_accounting": 0.5907473309608541, "philosophy": 0.8064516129032258, "professional_medicine": 0.8487084870848709, "nutrition": 0.8524590163934426, "global_facts": 0.6262626262626263, "machine_learning": 0.5405405405405406, "security_studies": 0.7459016393442623, "public_relations": 0.7155963302752294, "professional_psychology": 0.8101472995090017, "prehistory": 0.8668730650154799, "anatomy": 0.746268656716418, "human_sexuality": 0.8461538461538461, "college_medicine": 0.7674418604651163, "high_school_government_and_politics": 0.9479166666666666, "college_chemistry": 0.5252525252525253, "logical_fallacies": 0.8765432098765432, "high_school_geography": 0.8527918781725888, "elementary_mathematics": 0.8063660477453581, "human_aging": 0.8153153153153153, "college_mathematics": 0.35353535353535354, "high_school_psychology": 0.9025735294117647, "formal_logic": 0.616, "high_school_statistics": 0.6604651162790698, "international_law": 0.8916666666666667, "high_school_mathematics": 0.42379182156133827, "high_school_computer_science": 0.7878787878787878, "conceptual_physics": 0.7692307692307693, "miscellaneous": 0.9040920716112532, "high_school_chemistry": 0.6534653465346535, "marketing": 0.9484978540772532, "professional_law": 0.6099151989562949, "management": 0.8333333333333334, "college_physics": 0.5841584158415841, "jurisprudence": 0.8037383177570093, "world_religions": 0.888235294117647, "sociology": 0.88, "us_foreign_policy": 0.9191919191919192, "high_school_macroeconomics": 0.7506426735218509, "computer_security": 0.7878787878787878, "moral_scenarios": 0.6185682326621924, "moral_disputes": 0.8, "electrical_engineering": 0.7013888888888888, "astronomy": 0.8940397350993378, "college_biology": 0.8461538461538461 } }, "prompt_4": { "accuracy": 0.7461565963532356, "category_acc": { "high_school_european_history": 0.774390243902439, "business_ethics": 0.8282828282828283, "clinical_knowledge": 0.7462121212121212, "medical_genetics": 0.8787878787878788, "high_school_us_history": 0.8522167487684729, "high_school_physics": 0.6, "high_school_world_history": 0.8898305084745762, "virology": 0.5515151515151515, "high_school_microeconomics": 0.8354430379746836, "econometrics": 0.5486725663716814, "college_computer_science": 0.6161616161616161, "high_school_biology": 0.8122977346278317, "abstract_algebra": 0.5151515151515151, "professional_accounting": 0.6120996441281139, "philosophy": 0.8, "professional_medicine": 0.8191881918819188, "nutrition": 0.7901639344262295, "global_facts": 0.6767676767676768, "machine_learning": 0.5765765765765766, "security_studies": 0.7540983606557377, "public_relations": 0.7155963302752294, "professional_psychology": 0.8150572831423896, "prehistory": 0.8668730650154799, "anatomy": 0.7388059701492538, "human_sexuality": 0.8461538461538461, "college_medicine": 0.7151162790697675, "high_school_government_and_politics": 0.921875, "college_chemistry": 0.48484848484848486, "logical_fallacies": 0.8395061728395061, "high_school_geography": 0.8426395939086294, "elementary_mathematics": 0.830238726790451, "human_aging": 0.8018018018018018, "college_mathematics": 0.41414141414141414, "high_school_psychology": 0.8676470588235294, "formal_logic": 0.616, "high_school_statistics": 0.5767441860465117, "international_law": 0.875, "high_school_mathematics": 0.44981412639405205, "high_school_computer_science": 0.8585858585858586, "conceptual_physics": 0.7692307692307693, "miscellaneous": 0.9002557544757033, "high_school_chemistry": 0.6683168316831684, "marketing": 0.9098712446351931, "professional_law": 0.594259621656882, "management": 0.8235294117647058, "college_physics": 0.5445544554455446, "jurisprudence": 0.794392523364486, "world_religions": 0.8764705882352941, "sociology": 0.83, "us_foreign_policy": 0.898989898989899, "high_school_macroeconomics": 0.7377892030848329, "computer_security": 0.7272727272727273, "moral_scenarios": 0.6331096196868009, "moral_disputes": 0.7913043478260869, "electrical_engineering": 0.7361111111111112, "astronomy": 0.8874172185430463, "college_biology": 0.8251748251748252 } }, "prompt_5": { "accuracy": 0.6911691097604576, "category_acc": { "high_school_european_history": 0.774390243902439, "business_ethics": 0.7373737373737373, "clinical_knowledge": 0.6590909090909091, "medical_genetics": 0.7676767676767676, "high_school_us_history": 0.8620689655172413, "high_school_physics": 0.49333333333333335, "high_school_world_history": 0.8813559322033898, "virology": 0.509090909090909, "high_school_microeconomics": 0.7721518987341772, "econometrics": 0.5132743362831859, "college_computer_science": 0.5555555555555556, "high_school_biology": 0.7119741100323624, "abstract_algebra": 0.35353535353535354, "professional_accounting": 0.505338078291815, "philosophy": 0.7612903225806451, "professional_medicine": 0.8302583025830258, "nutrition": 0.7278688524590164, "global_facts": 0.6363636363636364, "machine_learning": 0.4954954954954955, "security_studies": 0.6762295081967213, "public_relations": 0.6146788990825688, "professional_psychology": 0.7086743044189853, "prehistory": 0.7770897832817337, "anatomy": 0.6343283582089553, "human_sexuality": 0.7615384615384615, "college_medicine": 0.6395348837209303, "high_school_government_and_politics": 0.8854166666666666, "college_chemistry": 0.4444444444444444, "logical_fallacies": 0.7839506172839507, "high_school_geography": 0.7563451776649747, "elementary_mathematics": 0.8063660477453581, "human_aging": 0.7072072072072072, "college_mathematics": 0.3333333333333333, "high_school_psychology": 0.7573529411764706, "formal_logic": 0.576, "high_school_statistics": 0.5441860465116279, "international_law": 0.8166666666666667, "high_school_mathematics": 0.34572490706319703, "high_school_computer_science": 0.7676767676767676, "conceptual_physics": 0.7222222222222222, "miscellaneous": 0.8772378516624041, "high_school_chemistry": 0.594059405940594, "marketing": 0.8111587982832618, "professional_law": 0.5727332028701891, "management": 0.7647058823529411, "college_physics": 0.42574257425742573, "jurisprudence": 0.8317757009345794, "world_religions": 0.8352941176470589, "sociology": 0.815, "us_foreign_policy": 0.8686868686868687, "high_school_macroeconomics": 0.6940874035989717, "computer_security": 0.696969696969697, "moral_scenarios": 0.6185682326621924, "moral_disputes": 0.7594202898550725, "electrical_engineering": 0.6666666666666666, "astronomy": 0.7814569536423841, "college_biology": 0.7552447552447552 } } }, "c_eval": { "prompt_1": { "accuracy": 0.6173848439821694 }, "prompt_2": { "accuracy": 0.5891530460624071 }, "prompt_3": { "accuracy": 0.6181277860326895 }, "prompt_4": { "accuracy": 0.4888558692421991 }, "prompt_5": { "accuracy": 0.5312035661218425 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.6145703611457036, "category_acc": { "computer_network": 0.7083333333333334, "operating_system": 0.75, "computer_architecture": 0.6923076923076923, "college_programming": 0.7142857142857143, "college_physics": 0.5833333333333334, "college_chemistry": 0.5172413793103449, "advanced_mathematics": 0.5416666666666666, "probability_and_statistics": 0.4782608695652174, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.6896551724137931, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.4166666666666667, "high_school_biology": 0.375, "middle_school_mathematics": 0.5, "middle_school_biology": 0.9615384615384616, "middle_school_physics": 0.5833333333333334, "middle_school_chemistry": 0.72, "veterinary_medicine": 0.8214285714285714, "college_economics": 0.6, "business_administration": 0.6052631578947368, "marxism": 0.6666666666666666, "mao_zedong_thought": 0.5862068965517241, "education_science": 0.7647058823529411, "teacher_qualification": 0.8571428571428571, "high_school_politics": 0.7083333333333334, "high_school_geography": 0.625, "middle_school_politics": 0.9615384615384616, "middle_school_geography": 0.7058823529411765, "modern_chinese_history": 0.5714285714285714, "ideological_and_moral_cultivation": 0.625, "logic": 0.7407407407407407, "law": 0.4482758620689655, "chinese_language_and_literature": 0.6071428571428571, "art_studies": 0.7105263157894737, "professional_tour_guide": 0.6470588235294118, "legal_professional": 0.35714285714285715, "high_school_chinese": 0.375, "high_school_history": 0.64, "middle_school_history": 0.7037037037037037, "civil_servant": 0.5, "sports_science": 0.6666666666666666, "plant_protection": 0.8148148148148148, "basic_medicine": 0.8333333333333334, "clinical_medicine": 0.5925925925925926, "urban_and_rural_planner": 0.5294117647058824, "accountant": 0.5925925925925926, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.5277777777777778, "tax_accountant": 0.5555555555555556, "physician": 0.7962962962962963 } }, "prompt_2": -1, "prompt_3": { "accuracy": 0.6201743462017435, "category_acc": { "computer_network": 0.75, "operating_system": 0.75, "computer_architecture": 0.7692307692307693, "college_programming": 0.7380952380952381, "college_physics": 0.6666666666666666, "college_chemistry": 0.5517241379310345, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.6086956521739131, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.47619047619047616, "metrology_engineer": 0.6206896551724138, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.5416666666666666, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.5, "middle_school_biology": 0.8846153846153846, "middle_school_physics": 0.5, "middle_school_chemistry": 0.76, "veterinary_medicine": 0.8214285714285714, "college_economics": 0.6, "business_administration": 0.6842105263157895, "marxism": 0.7083333333333334, "mao_zedong_thought": 0.7586206896551724, "education_science": 0.7352941176470589, "teacher_qualification": 0.8367346938775511, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.8333333333333334, "middle_school_politics": 0.9230769230769231, "middle_school_geography": 0.7647058823529411, "modern_chinese_history": 0.6785714285714286, "ideological_and_moral_cultivation": 0.7083333333333334, "logic": 0.5925925925925926, "law": 0.5862068965517241, "chinese_language_and_literature": 0.5714285714285714, "art_studies": 0.7105263157894737, "professional_tour_guide": 0.6470588235294118, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.375, "high_school_history": 0.76, "middle_school_history": 0.6666666666666666, "civil_servant": 0.5192307692307693, "sports_science": 0.5833333333333334, "plant_protection": 0.8518518518518519, "basic_medicine": 0.75, "clinical_medicine": 0.6296296296296297, "urban_and_rural_planner": 0.45098039215686275, "accountant": 0.5925925925925926, "fire_engineer": 0.4166666666666667, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.46296296296296297, "physician": 0.7037037037037037 } }, "prompt_4": { "accuracy": 0.4887920298879203, "category_acc": { "computer_network": 0.625, "operating_system": 0.7083333333333334, "computer_architecture": 0.5, "college_programming": 0.47619047619047616, "college_physics": 0.3333333333333333, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.38095238095238093, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.4166666666666667, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.6153846153846154, "middle_school_physics": 0.5, "middle_school_chemistry": 0.6, "veterinary_medicine": 0.5, "college_economics": 0.35, "business_administration": 0.5, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.6206896551724138, "education_science": 0.47058823529411764, "teacher_qualification": 0.6938775510204082, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.5, "middle_school_politics": 0.8076923076923077, "middle_school_geography": 0.7647058823529411, "modern_chinese_history": 0.5714285714285714, "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.6296296296296297, "law": 0.41379310344827586, "chinese_language_and_literature": 0.6071428571428571, "art_studies": 0.5, "professional_tour_guide": 0.5294117647058824, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.4583333333333333, "high_school_history": 0.44, "middle_school_history": 0.6666666666666666, "civil_servant": 0.5, "sports_science": 0.5416666666666666, "plant_protection": 0.7037037037037037, "basic_medicine": 0.75, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.39215686274509803, "accountant": 0.42592592592592593, "fire_engineer": 0.3888888888888889, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.35185185185185186, "physician": 0.6851851851851852 } }, "prompt_5": { "accuracy": 0.5354919053549191, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.5833333333333334, "computer_architecture": 0.5, "college_programming": 0.5714285714285714, "college_physics": 0.3333333333333333, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.25, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.47619047619047616, "metrology_engineer": 0.7586206896551724, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.375, "high_school_chemistry": 0.4166666666666667, "high_school_biology": 0.375, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.6923076923076923, "middle_school_physics": 0.5416666666666666, "middle_school_chemistry": 0.48, "veterinary_medicine": 0.6785714285714286, "college_economics": 0.5666666666666667, "business_administration": 0.47368421052631576, "marxism": 0.7916666666666666, "mao_zedong_thought": 0.6896551724137931, "education_science": 0.6764705882352942, "teacher_qualification": 0.7755102040816326, "high_school_politics": 0.5833333333333334, "high_school_geography": 0.7083333333333334, "middle_school_politics": 0.8461538461538461, "middle_school_geography": 0.7647058823529411, "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.625, "logic": 0.5555555555555556, "law": 0.3793103448275862, "chinese_language_and_literature": 0.5357142857142857, "art_studies": 0.631578947368421, "professional_tour_guide": 0.6176470588235294, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.4583333333333333, "high_school_history": 0.6, "middle_school_history": 0.7407407407407407, "civil_servant": 0.5192307692307693, "sports_science": 0.5833333333333334, "plant_protection": 0.7407407407407407, "basic_medicine": 0.6666666666666666, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.43137254901960786, "accountant": 0.48148148148148145, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.6388888888888888, "tax_accountant": 0.35185185185185186, "physician": 0.7037037037037037 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.6451612903225806 }, "prompt_2": { "accuracy": 0.6487455197132617 }, "prompt_3": { "accuracy": 0.6308243727598566 }, "prompt_4": { "accuracy": 0.4910394265232975 }, "prompt_5": { "accuracy": 0.6308243727598566 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.6396995337592817, "category_acc": { "agronomy": 0.6745562130177515, "anatomy": 0.6351351351351351, "ancient_chinese": 0.3719512195121951, "arts": 0.76875, "astronomy": 0.4727272727272727, "business_ethics": 0.583732057416268, "chinese_civil_service_exam": 0.625, "chinese_driving_rule": 0.7175572519083969, "chinese_food_culture": 0.5514705882352942, "chinese_foreign_policy": 0.616822429906542, "chinese_history": 0.7151702786377709, "chinese_literature": 0.4362745098039216, "chinese_teacher_qualification": 0.7597765363128491, "clinical_knowledge": 0.7215189873417721, "college_actuarial_science": 0.3018867924528302, "college_education": 0.6728971962616822, "college_engineering_hydrology": 0.5849056603773585, "college_law": 0.5092592592592593, "college_mathematics": 0.4095238095238095, "college_medical_statistics": 0.6509433962264151, "college_medicine": 0.7289377289377289, "computer_science": 0.7696078431372549, "computer_security": 0.8362573099415205, "conceptual_physics": 0.5374149659863946, "construction_project_management": 0.45323741007194246, "economics": 0.7232704402515723, "education": 0.6319018404907976, "electrical_engineering": 0.6337209302325582, "elementary_chinese": 0.5079365079365079, "elementary_commonsense": 0.6111111111111112, "elementary_information_and_technology": 0.8907563025210085, "elementary_mathematics": 0.4956521739130435, "ethnology": 0.6370370370370371, "food_science": 0.6153846153846154, "genetics": 0.5852272727272727, "global_facts": 0.697986577181208, "high_school_biology": 0.5443786982248521, "high_school_chemistry": 0.4090909090909091, "high_school_geography": 0.6186440677966102, "high_school_mathematics": 0.42073170731707316, "high_school_physics": 0.6090909090909091, "high_school_politics": 0.6223776223776224, "human_sexuality": 0.6746031746031746, "international_law": 0.6810810810810811, "journalism": 0.6453488372093024, "jurisprudence": 0.6082725060827251, "legal_and_moral_basis": 0.8551401869158879, "logical": 0.6666666666666666, "machine_learning": 0.6639344262295082, "management": 0.7714285714285715, "marketing": 0.7888888888888889, "marxist_theory": 0.6825396825396826, "modern_chinese": 0.45689655172413796, "nutrition": 0.6482758620689655, "philosophy": 0.6476190476190476, "professional_accounting": 0.6685714285714286, "professional_law": 0.45023696682464454, "professional_medicine": 0.6808510638297872, "professional_psychology": 0.7370689655172413, "public_relations": 0.5919540229885057, "security_study": 0.7851851851851852, "sociology": 0.6637168141592921, "sports_science": 0.696969696969697, "traditional_chinese_medicine": 0.5891891891891892, "virology": 0.7218934911242604, "world_history": 0.7453416149068323, "world_religions": 0.775 } }, "prompt_2": { "accuracy": 0.632360559488862, "category_acc": { "agronomy": 0.6331360946745562, "anatomy": 0.6418918918918919, "ancient_chinese": 0.3597560975609756, "arts": 0.76875, "astronomy": 0.43636363636363634, "business_ethics": 0.6555023923444976, "chinese_civil_service_exam": 0.525, "chinese_driving_rule": 0.8244274809160306, "chinese_food_culture": 0.5441176470588235, "chinese_foreign_policy": 0.6728971962616822, "chinese_history": 0.7120743034055728, "chinese_literature": 0.45098039215686275, "chinese_teacher_qualification": 0.7541899441340782, "clinical_knowledge": 0.7215189873417721, "college_actuarial_science": 0.3113207547169811, "college_education": 0.7009345794392523, "college_engineering_hydrology": 0.6037735849056604, "college_law": 0.49074074074074076, "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.6415094339622641, "college_medicine": 0.673992673992674, "computer_science": 0.7401960784313726, "computer_security": 0.7368421052631579, "conceptual_physics": 0.54421768707483, "construction_project_management": 0.43884892086330934, "economics": 0.6981132075471698, "education": 0.6441717791411042, "electrical_engineering": 0.622093023255814, "elementary_chinese": 0.503968253968254, "elementary_commonsense": 0.6414141414141414, "elementary_information_and_technology": 0.865546218487395, "elementary_mathematics": 0.4652173913043478, "ethnology": 0.6518518518518519, "food_science": 0.5804195804195804, "genetics": 0.5625, "global_facts": 0.6912751677852349, "high_school_biology": 0.4319526627218935, "high_school_chemistry": 0.3560606060606061, "high_school_geography": 0.635593220338983, "high_school_mathematics": 0.32926829268292684, "high_school_physics": 0.5545454545454546, "high_school_politics": 0.6783216783216783, "human_sexuality": 0.6587301587301587, "international_law": 0.6972972972972973, "journalism": 0.6511627906976745, "jurisprudence": 0.5863746958637469, "legal_and_moral_basis": 0.8738317757009346, "logical": 0.6178861788617886, "machine_learning": 0.5983606557377049, "management": 0.7857142857142857, "marketing": 0.7444444444444445, "marxist_theory": 0.6984126984126984, "modern_chinese": 0.5, "nutrition": 0.6413793103448275, "philosophy": 0.6666666666666666, "professional_accounting": 0.6628571428571428, "professional_law": 0.47393364928909953, "professional_medicine": 0.6861702127659575, "professional_psychology": 0.75, "public_relations": 0.6264367816091954, "security_study": 0.7925925925925926, "sociology": 0.7035398230088495, "sports_science": 0.6424242424242425, "traditional_chinese_medicine": 0.6756756756756757, "virology": 0.7337278106508875, "world_history": 0.7142857142857143, "world_religions": 0.76875 } }, "prompt_3": { "accuracy": 0.6512692108444137, "category_acc": { "agronomy": 0.6923076923076923, "anatomy": 0.5878378378378378, "ancient_chinese": 0.3719512195121951, "arts": 0.75, "astronomy": 0.43636363636363634, "business_ethics": 0.6555023923444976, "chinese_civil_service_exam": 0.5875, "chinese_driving_rule": 0.7862595419847328, "chinese_food_culture": 0.5661764705882353, "chinese_foreign_policy": 0.7009345794392523, "chinese_history": 0.7027863777089783, "chinese_literature": 0.44607843137254904, "chinese_teacher_qualification": 0.770949720670391, "clinical_knowledge": 0.7257383966244726, "college_actuarial_science": 0.3867924528301887, "college_education": 0.8037383177570093, "college_engineering_hydrology": 0.6509433962264151, "college_law": 0.5370370370370371, "college_mathematics": 0.42857142857142855, "college_medical_statistics": 0.6886792452830188, "college_medicine": 0.7252747252747253, "computer_science": 0.7794117647058824, "computer_security": 0.7719298245614035, "conceptual_physics": 0.46258503401360546, "construction_project_management": 0.4028776978417266, "economics": 0.7421383647798742, "education": 0.6625766871165644, "electrical_engineering": 0.6569767441860465, "elementary_chinese": 0.5198412698412699, "elementary_commonsense": 0.6313131313131313, "elementary_information_and_technology": 0.8865546218487395, "elementary_mathematics": 0.5304347826086957, "ethnology": 0.6666666666666666, "food_science": 0.5944055944055944, "genetics": 0.6022727272727273, "global_facts": 0.697986577181208, "high_school_biology": 0.514792899408284, "high_school_chemistry": 0.4318181818181818, "high_school_geography": 0.6694915254237288, "high_school_mathematics": 0.38414634146341464, "high_school_physics": 0.5454545454545454, "high_school_politics": 0.6643356643356644, "human_sexuality": 0.7222222222222222, "international_law": 0.6702702702702703, "journalism": 0.6976744186046512, "jurisprudence": 0.6058394160583942, "legal_and_moral_basis": 0.883177570093458, "logical": 0.6747967479674797, "machine_learning": 0.5819672131147541, "management": 0.7952380952380952, "marketing": 0.7722222222222223, "marxist_theory": 0.7195767195767195, "modern_chinese": 0.5086206896551724, "nutrition": 0.6068965517241379, "philosophy": 0.6761904761904762, "professional_accounting": 0.7028571428571428, "professional_law": 0.46919431279620855, "professional_medicine": 0.6781914893617021, "professional_psychology": 0.7801724137931034, "public_relations": 0.632183908045977, "security_study": 0.8296296296296296, "sociology": 0.6769911504424779, "sports_science": 0.696969696969697, "traditional_chinese_medicine": 0.6702702702702703, "virology": 0.7337278106508875, "world_history": 0.7639751552795031, "world_religions": 0.79375 } }, "prompt_4": { "accuracy": 0.5289241927128302, "category_acc": { "agronomy": 0.5621301775147929, "anatomy": 0.4391891891891892, "ancient_chinese": 0.3353658536585366, "arts": 0.6125, "astronomy": 0.40606060606060607, "business_ethics": 0.45933014354066987, "chinese_civil_service_exam": 0.49375, "chinese_driving_rule": 0.6259541984732825, "chinese_food_culture": 0.5294117647058824, "chinese_foreign_policy": 0.5327102803738317, "chinese_history": 0.5758513931888545, "chinese_literature": 0.4068627450980392, "chinese_teacher_qualification": 0.6368715083798883, "clinical_knowledge": 0.6033755274261603, "college_actuarial_science": 0.20754716981132076, "college_education": 0.5981308411214953, "college_engineering_hydrology": 0.5660377358490566, "college_law": 0.4351851851851852, "college_mathematics": 0.23809523809523808, "college_medical_statistics": 0.4811320754716981, "college_medicine": 0.5970695970695971, "computer_science": 0.6078431372549019, "computer_security": 0.631578947368421, "conceptual_physics": 0.4217687074829932, "construction_project_management": 0.4460431654676259, "economics": 0.5911949685534591, "education": 0.5337423312883436, "electrical_engineering": 0.5, "elementary_chinese": 0.4603174603174603, "elementary_commonsense": 0.5606060606060606, "elementary_information_and_technology": 0.7142857142857143, "elementary_mathematics": 0.43043478260869567, "ethnology": 0.5777777777777777, "food_science": 0.4965034965034965, "genetics": 0.48295454545454547, "global_facts": 0.5570469798657718, "high_school_biology": 0.44970414201183434, "high_school_chemistry": 0.3484848484848485, "high_school_geography": 0.4915254237288136, "high_school_mathematics": 0.27439024390243905, "high_school_physics": 0.36363636363636365, "high_school_politics": 0.4825174825174825, "human_sexuality": 0.5079365079365079, "international_law": 0.6, "journalism": 0.5406976744186046, "jurisprudence": 0.5012165450121655, "legal_and_moral_basis": 0.6962616822429907, "logical": 0.5040650406504065, "machine_learning": 0.5573770491803278, "management": 0.6095238095238096, "marketing": 0.6, "marxist_theory": 0.5238095238095238, "modern_chinese": 0.41379310344827586, "nutrition": 0.5862068965517241, "philosophy": 0.6285714285714286, "professional_accounting": 0.49714285714285716, "professional_law": 0.5071090047393365, "professional_medicine": 0.5478723404255319, "professional_psychology": 0.5818965517241379, "public_relations": 0.4942528735632184, "security_study": 0.7185185185185186, "sociology": 0.584070796460177, "sports_science": 0.5333333333333333, "traditional_chinese_medicine": 0.5513513513513514, "virology": 0.5976331360946746, "world_history": 0.5590062111801242, "world_religions": 0.7125 } }, "prompt_5": { "accuracy": 0.5649283370747712, "category_acc": { "agronomy": 0.6390532544378699, "anatomy": 0.5337837837837838, "ancient_chinese": 0.3048780487804878, "arts": 0.6625, "astronomy": 0.42424242424242425, "business_ethics": 0.5215311004784688, "chinese_civil_service_exam": 0.48125, "chinese_driving_rule": 0.7557251908396947, "chinese_food_culture": 0.5073529411764706, "chinese_foreign_policy": 0.6074766355140186, "chinese_history": 0.6377708978328174, "chinese_literature": 0.4068627450980392, "chinese_teacher_qualification": 0.6871508379888268, "clinical_knowledge": 0.6286919831223629, "college_actuarial_science": 0.2641509433962264, "college_education": 0.7289719626168224, "college_engineering_hydrology": 0.5943396226415094, "college_law": 0.4722222222222222, "college_mathematics": 0.20952380952380953, "college_medical_statistics": 0.6415094339622641, "college_medicine": 0.6227106227106227, "computer_science": 0.6274509803921569, "computer_security": 0.6081871345029239, "conceptual_physics": 0.4557823129251701, "construction_project_management": 0.4316546762589928, "economics": 0.559748427672956, "education": 0.6134969325153374, "electrical_engineering": 0.5872093023255814, "elementary_chinese": 0.44841269841269843, "elementary_commonsense": 0.5606060606060606, "elementary_information_and_technology": 0.7815126050420168, "elementary_mathematics": 0.4434782608695652, "ethnology": 0.6148148148148148, "food_science": 0.5244755244755245, "genetics": 0.4943181818181818, "global_facts": 0.6174496644295302, "high_school_biology": 0.39644970414201186, "high_school_chemistry": 0.30303030303030304, "high_school_geography": 0.5169491525423728, "high_school_mathematics": 0.29878048780487804, "high_school_physics": 0.39090909090909093, "high_school_politics": 0.5734265734265734, "human_sexuality": 0.626984126984127, "international_law": 0.6054054054054054, "journalism": 0.5813953488372093, "jurisprudence": 0.5231143552311436, "legal_and_moral_basis": 0.7990654205607477, "logical": 0.5365853658536586, "machine_learning": 0.5819672131147541, "management": 0.7428571428571429, "marketing": 0.6833333333333333, "marxist_theory": 0.6402116402116402, "modern_chinese": 0.3793103448275862, "nutrition": 0.6068965517241379, "philosophy": 0.580952380952381, "professional_accounting": 0.56, "professional_law": 0.4549763033175355, "professional_medicine": 0.5531914893617021, "professional_psychology": 0.6422413793103449, "public_relations": 0.6206896551724138, "security_study": 0.7407407407407407, "sociology": 0.5707964601769911, "sports_science": 0.5757575757575758, "traditional_chinese_medicine": 0.6162162162162163, "virology": 0.6272189349112426, "world_history": 0.6645962732919255, "world_religions": 0.70625 } } }, "zbench": { "prompt_1": { "accuracy": 0.42424242424242425 }, "prompt_2": { "accuracy": 0.42424242424242425 }, "prompt_3": { "accuracy": 0.48484848484848486 }, "prompt_4": { "accuracy": 0.30303030303030304 }, "prompt_5": { "accuracy": 0.42424242424242425 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6886363636363636 }, "prompt_2": { "accuracy": 0.6727272727272727 }, "prompt_3": { "accuracy": 0.6863636363636364 }, "prompt_4": { "accuracy": 0.6704545454545454 }, "prompt_5": { "accuracy": 0.6795454545454546 } }, "ocnli": { "prompt_1": { "accuracy": 0.644406779661017 }, "prompt_2": { "accuracy": 0.6491525423728813 }, "prompt_3": { "accuracy": 0.6610169491525424 }, "prompt_4": { "accuracy": 0.5938983050847457 }, "prompt_5": { "accuracy": 0.42338983050847456 } }, "c3": { "prompt_1": { "accuracy": 0.949513836948392 }, "prompt_2": { "accuracy": 0.9513836948391923 }, "prompt_3": { "accuracy": 0.9420344053851907 }, "prompt_4": { "accuracy": 0.9525056095736724 }, "prompt_5": { "accuracy": 0.9517576664173523 } }, "dream": { "prompt_1": { "accuracy": 0.9603135717785399 }, "prompt_2": { "accuracy": 0.9524742773150416 }, "prompt_3": { "accuracy": 0.9573738363547281 }, "prompt_4": { "accuracy": 0.9563939245467908 }, "prompt_5": { "accuracy": 0.953454189122979 } }, "samsum": { "prompt_1": { "rouge1": 0.2331052073691148, "rouge2": 0.08833064572340593, "rougeL": 0.18262886117554988, "avg_rouge": 0.16802157142269022 }, "prompt_2": { "rouge1": 0.22079406450584757, "rouge2": 0.08256077131086058, "rougeL": 0.1773810197063271, "avg_rouge": 0.16024528517434508 }, "prompt_3": { "rouge1": 0.22311189627879652, "rouge2": 0.08051561232432305, "rougeL": 0.17507550912142414, "avg_rouge": 0.1595676725748479 }, "prompt_4": { "rouge1": 0.22027271695407136, "rouge2": 0.08217846617891715, "rougeL": 0.17684600215667332, "avg_rouge": 0.15976572842988726 }, "prompt_5": { "rouge1": 0.21139709311099802, "rouge2": 0.07271829499360966, "rougeL": 0.1714907858235061, "avg_rouge": 0.1518687246427046 } }, "dialogsum": { "prompt_1": { "rouge1": 0.14272835035532452, "rouge2": 0.031426216985853635, "rougeL": 0.10874297216711339, "avg_rouge": 0.09429917983609719 }, "prompt_2": { "rouge1": 0.18598054718281445, "rouge2": 0.045910564656826275, "rougeL": 0.13597379845717178, "avg_rouge": 0.12262163676560417 }, "prompt_3": { "rouge1": 0.17458000087891037, "rouge2": 0.04185218070044238, "rougeL": 0.1286382832296819, "avg_rouge": 0.11502348826967823 }, "prompt_4": { "rouge1": 0.17935557506829655, "rouge2": 0.03903281064676857, "rougeL": 0.13073031041529273, "avg_rouge": 0.11637289871011929 }, "prompt_5": { "rouge1": 0.1990788358884124, "rouge2": 0.04801377685941248, "rougeL": 0.14486307743368151, "avg_rouge": 0.13065189672716881 } }, "sst2": { "prompt_1": { "accuracy": 0.9529816513761468 }, "prompt_2": { "accuracy": 0.9610091743119266 }, "prompt_3": { "accuracy": 0.9438073394495413 }, "prompt_4": { "accuracy": 0.948394495412844 }, "prompt_5": { "accuracy": 0.9243119266055045 } }, "cola": { "prompt_1": { "accuracy": 0.7919463087248322 }, "prompt_2": { "accuracy": 0.775647171620326 }, "prompt_3": { "accuracy": 0.7833173537871524 }, "prompt_4": { "accuracy": 0.7919463087248322 }, "prompt_5": { "accuracy": 0.8235858101629914 } }, "qqp": { "prompt_1": { "accuracy": 0.8165 }, "prompt_2": { "accuracy": 0.7715 }, "prompt_3": { "accuracy": 0.8015 }, "prompt_4": { "accuracy": 0.818 }, "prompt_5": { "accuracy": 0.777 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.6 } }, "qnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 1.0 } }, "wnli": { "prompt_1": { "accuracy": 1.0 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "sea_lion_3b": { "model_size": "3B", "model_link": "https://huggingface.co/aisingapore/sea-lion-3b", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.26512605042016807, "language_acc": { "Vietnamese": 0.2739495798319328, "Spanish": 0.23277310924369748, "English": 0.27983193277310925, "Chinese": 0.2739495798319328 }, "consistency_score_2": 0.40574229691876745, "consistency_score_3": 0.20546218487394957, "consistency_score_4": 0.11428571428571428, "detailed_consistency_score": { "2_combine": { "Vietnamese,Spanish": 0.419327731092437, "Vietnamese,English": 0.4579831932773109, "Vietnamese,Chinese": 0.37563025210084033, "Spanish,English": 0.41596638655462187, "Spanish,Chinese": 0.3630252100840336, "English,Chinese": 0.40252100840336136 }, "3_combine": { "Vietnamese,Spanish,English": 0.23529411764705882, "Vietnamese,Spanish,Chinese": 0.18571428571428572, "Vietnamese,English,Chinese": 0.20840336134453782, "Spanish,English,Chinese": 0.19243697478991598 }, "4_combine": { "Vietnamese,Spanish,English,Chinese": 0.11428571428571428 } }, "AC3_2": 0.32069735613893025, "AC3_3": 0.2315118546927005, "AC3_4": 0.15972156300172502 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.23523809523809527, "language_acc": { "Chinese": 0.2, "Indonesian": 0.2866666666666667, "Spanish": 0.20666666666666667, "Vietnamese": 0.24666666666666667, "Malay": 0.22, "English": 0.25333333333333335, "Filipino": 0.23333333333333334 }, "consistency_score_2": 0.5136507936507937, "consistency_score_3": 0.3165714285714285, "consistency_score_4": 0.21409523809523806, "consistency_score_5": 0.15269841269841272, "consistency_score_6": 0.11333333333333331, "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { "Chinese,Indonesian": 0.43333333333333335, "Chinese,Spanish": 0.49333333333333335, "Chinese,Vietnamese": 0.46, "Chinese,Malay": 0.5, "Chinese,English": 0.47333333333333333, "Chinese,Filipino": 0.44666666666666666, "Indonesian,Spanish": 0.5, "Indonesian,Vietnamese": 0.5, "Indonesian,Malay": 0.5466666666666666, "Indonesian,English": 0.5, "Indonesian,Filipino": 0.54, "Spanish,Vietnamese": 0.5133333333333333, "Spanish,Malay": 0.5666666666666667, "Spanish,English": 0.5133333333333333, "Spanish,Filipino": 0.5266666666666666, "Vietnamese,Malay": 0.5466666666666666, "Vietnamese,English": 0.48, "Vietnamese,Filipino": 0.5466666666666666, "Malay,English": 0.58, "Malay,Filipino": 0.5666666666666667, "English,Filipino": 0.5533333333333333 }, "3_combine": { "Chinese,Indonesian,Spanish": 0.26, "Chinese,Indonesian,Vietnamese": 0.26666666666666666, "Chinese,Indonesian,Malay": 0.29333333333333333, "Chinese,Indonesian,English": 0.26, "Chinese,Indonesian,Filipino": 0.25333333333333335, "Chinese,Spanish,Vietnamese": 0.28, "Chinese,Spanish,Malay": 0.32, "Chinese,Spanish,English": 0.3, "Chinese,Spanish,Filipino": 0.28, "Chinese,Vietnamese,Malay": 0.29333333333333333, "Chinese,Vietnamese,English": 0.26666666666666666, "Chinese,Vietnamese,Filipino": 0.2866666666666667, "Chinese,Malay,English": 0.32666666666666666, "Chinese,Malay,Filipino": 0.3, "Chinese,English,Filipino": 0.28, "Indonesian,Spanish,Vietnamese": 0.3, "Indonesian,Spanish,Malay": 0.3333333333333333, "Indonesian,Spanish,English": 0.30666666666666664, "Indonesian,Spanish,Filipino": 0.3333333333333333, "Indonesian,Vietnamese,Malay": 0.3466666666666667, "Indonesian,Vietnamese,English": 0.3, "Indonesian,Vietnamese,Filipino": 0.34, "Indonesian,Malay,English": 0.35333333333333333, "Indonesian,Malay,Filipino": 0.36, "Indonesian,English,Filipino": 0.35333333333333333, "Spanish,Vietnamese,Malay": 0.3466666666666667, "Spanish,Vietnamese,English": 0.3, "Spanish,Vietnamese,Filipino": 0.34, "Spanish,Malay,English": 0.36666666666666664, "Spanish,Malay,Filipino": 0.36, "Spanish,English,Filipino": 0.34, "Vietnamese,Malay,English": 0.3466666666666667, "Vietnamese,Malay,Filipino": 0.36666666666666664, "Vietnamese,English,Filipino": 0.32666666666666666, "Malay,English,Filipino": 0.3933333333333333 }, "4_combine": { "Chinese,Indonesian,Spanish,Vietnamese": 0.17333333333333334, "Chinese,Indonesian,Spanish,Malay": 0.19333333333333333, "Chinese,Indonesian,Spanish,English": 0.18666666666666668, "Chinese,Indonesian,Spanish,Filipino": 0.18666666666666668, "Chinese,Indonesian,Vietnamese,Malay": 0.2, "Chinese,Indonesian,Vietnamese,English": 0.18, "Chinese,Indonesian,Vietnamese,Filipino": 0.17333333333333334, "Chinese,Indonesian,Malay,English": 0.21333333333333335, "Chinese,Indonesian,Malay,Filipino": 0.18666666666666668, "Chinese,Indonesian,English,Filipino": 0.18, "Chinese,Spanish,Vietnamese,Malay": 0.19333333333333333, "Chinese,Spanish,Vietnamese,English": 0.17333333333333334, "Chinese,Spanish,Vietnamese,Filipino": 0.19333333333333333, "Chinese,Spanish,Malay,English": 0.23333333333333334, "Chinese,Spanish,Malay,Filipino": 0.2, "Chinese,Spanish,English,Filipino": 0.21333333333333335, "Chinese,Vietnamese,Malay,English": 0.22, "Chinese,Vietnamese,Malay,Filipino": 0.20666666666666667, "Chinese,Vietnamese,English,Filipino": 0.19333333333333333, "Chinese,Malay,English,Filipino": 0.23333333333333334, "Indonesian,Spanish,Vietnamese,Malay": 0.22, "Indonesian,Spanish,Vietnamese,English": 0.18666666666666668, "Indonesian,Spanish,Vietnamese,Filipino": 0.23333333333333334, "Indonesian,Spanish,Malay,English": 0.24, "Indonesian,Spanish,Malay,Filipino": 0.23333333333333334, "Indonesian,Spanish,English,Filipino": 0.23333333333333334, "Indonesian,Vietnamese,Malay,English": 0.23333333333333334, "Indonesian,Vietnamese,Malay,Filipino": 0.25333333333333335, "Indonesian,Vietnamese,English,Filipino": 0.22666666666666666, "Indonesian,Malay,English,Filipino": 0.26666666666666666, "Spanish,Vietnamese,Malay,English": 0.23333333333333334, "Spanish,Vietnamese,Malay,Filipino": 0.25333333333333335, "Spanish,Vietnamese,English,Filipino": 0.22, "Spanish,Malay,English,Filipino": 0.26, "Vietnamese,Malay,English,Filipino": 0.26666666666666666 }, "5_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay": 0.13333333333333333, "Chinese,Indonesian,Spanish,Vietnamese,English": 0.12, "Chinese,Indonesian,Spanish,Vietnamese,Filipino": 0.13333333333333333, "Chinese,Indonesian,Spanish,Malay,English": 0.16, "Chinese,Indonesian,Spanish,Malay,Filipino": 0.13333333333333333, "Chinese,Indonesian,Spanish,English,Filipino": 0.14666666666666667, "Chinese,Indonesian,Vietnamese,Malay,English": 0.15333333333333332, "Chinese,Indonesian,Vietnamese,Malay,Filipino": 0.14, "Chinese,Indonesian,Vietnamese,English,Filipino": 0.12666666666666668, "Chinese,Indonesian,Malay,English,Filipino": 0.15333333333333332, "Chinese,Spanish,Vietnamese,Malay,English": 0.14666666666666667, "Chinese,Spanish,Vietnamese,Malay,Filipino": 0.14, "Chinese,Spanish,Vietnamese,English,Filipino": 0.14, "Chinese,Spanish,Malay,English,Filipino": 0.17333333333333334, "Chinese,Vietnamese,Malay,English,Filipino": 0.17333333333333334, "Indonesian,Spanish,Vietnamese,Malay,English": 0.15333333333333332, "Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.17333333333333334, "Indonesian,Spanish,Vietnamese,English,Filipino": 0.15333333333333332, "Indonesian,Spanish,Malay,English,Filipino": 0.18, "Indonesian,Vietnamese,Malay,English,Filipino": 0.18666666666666668, "Spanish,Vietnamese,Malay,English,Filipino": 0.18666666666666668 }, "6_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English": 0.10666666666666667, "Chinese,Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.1, "Chinese,Indonesian,Spanish,Vietnamese,English,Filipino": 0.1, "Chinese,Indonesian,Spanish,Malay,English,Filipino": 0.12, "Chinese,Indonesian,Vietnamese,Malay,English,Filipino": 0.11333333333333333, "Chinese,Spanish,Vietnamese,Malay,English,Filipino": 0.12, "Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.13333333333333333 }, "7_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.08666666666666667 } }, "AC3_2": 0.32269202038525996, "AC3_3": 0.2699107450568607, "AC3_4": 0.224169240346566, "AC3_5": 0.18518743662906187, "AC3_6": 0.15296903456449432, "AC3_7": 0.12666666662731757 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.25487012987012986, "language_acc": { "Vietnamese": 0.23295454545454544, "Indonesian": 0.2840909090909091, "Spanish": 0.25, "Malay": 0.26136363636363635, "Filipino": 0.23863636363636365, "English": 0.26704545454545453, "Chinese": 0.25 }, "consistency_score_2": 0.5435606060606061, "consistency_score_3": 0.32905844155844166, "consistency_score_4": 0.21347402597402604, "consistency_score_5": 0.14583333333333331, "consistency_score_6": 0.1030844155844156, "consistency_score_7": 0.07386363636363637, "detailed_consistency_score": { "2_combine": { "Vietnamese,Indonesian": 0.5852272727272727, "Vietnamese,Spanish": 0.48863636363636365, "Vietnamese,Malay": 0.5738636363636364, "Vietnamese,Filipino": 0.5568181818181818, "Vietnamese,English": 0.5340909090909091, "Vietnamese,Chinese": 0.5965909090909091, "Indonesian,Spanish": 0.5170454545454546, "Indonesian,Malay": 0.5170454545454546, "Indonesian,Filipino": 0.5511363636363636, "Indonesian,English": 0.5965909090909091, "Indonesian,Chinese": 0.5681818181818182, "Spanish,Malay": 0.5056818181818182, "Spanish,Filipino": 0.4318181818181818, "Spanish,English": 0.5056818181818182, "Spanish,Chinese": 0.4772727272727273, "Malay,Filipino": 0.5454545454545454, "Malay,English": 0.5454545454545454, "Malay,Chinese": 0.6363636363636364, "Filipino,English": 0.5397727272727273, "Filipino,Chinese": 0.5795454545454546, "English,Chinese": 0.5625 }, "3_combine": { "Vietnamese,Indonesian,Spanish": 0.30113636363636365, "Vietnamese,Indonesian,Malay": 0.3409090909090909, "Vietnamese,Indonesian,Filipino": 0.35795454545454547, "Vietnamese,Indonesian,English": 0.36363636363636365, "Vietnamese,Indonesian,Chinese": 0.38636363636363635, "Vietnamese,Spanish,Malay": 0.2897727272727273, "Vietnamese,Spanish,Filipino": 0.25, "Vietnamese,Spanish,English": 0.2840909090909091, "Vietnamese,Spanish,Chinese": 0.3068181818181818, "Vietnamese,Malay,Filipino": 0.3465909090909091, "Vietnamese,Malay,English": 0.3352272727272727, "Vietnamese,Malay,Chinese": 0.42045454545454547, "Vietnamese,Filipino,English": 0.3352272727272727, "Vietnamese,Filipino,Chinese": 0.38636363636363635, "Vietnamese,English,Chinese": 0.3693181818181818, "Indonesian,Spanish,Malay": 0.2784090909090909, "Indonesian,Spanish,Filipino": 0.26704545454545453, "Indonesian,Spanish,English": 0.32386363636363635, "Indonesian,Spanish,Chinese": 0.29545454545454547, "Indonesian,Malay,Filipino": 0.3181818181818182, "Indonesian,Malay,English": 0.3465909090909091, "Indonesian,Malay,Chinese": 0.3693181818181818, "Indonesian,Filipino,English": 0.35795454545454547, "Indonesian,Filipino,Chinese": 0.35795454545454547, "Indonesian,English,Chinese": 0.38636363636363635, "Spanish,Malay,Filipino": 0.2556818181818182, "Spanish,Malay,English": 0.29545454545454547, "Spanish,Malay,Chinese": 0.32386363636363635, "Spanish,Filipino,English": 0.24431818181818182, "Spanish,Filipino,Chinese": 0.26704545454545453, "Spanish,English,Chinese": 0.2784090909090909, "Malay,Filipino,English": 0.3352272727272727, "Malay,Filipino,Chinese": 0.3977272727272727, "Malay,English,Chinese": 0.3806818181818182, "Filipino,English,Chinese": 0.36363636363636365 }, "4_combine": { "Vietnamese,Indonesian,Spanish,Malay": 0.18181818181818182, "Vietnamese,Indonesian,Spanish,Filipino": 0.17613636363636365, "Vietnamese,Indonesian,Spanish,English": 0.20454545454545456, "Vietnamese,Indonesian,Spanish,Chinese": 0.21022727272727273, "Vietnamese,Indonesian,Malay,Filipino": 0.2215909090909091, "Vietnamese,Indonesian,Malay,English": 0.2215909090909091, "Vietnamese,Indonesian,Malay,Chinese": 0.26704545454545453, "Vietnamese,Indonesian,Filipino,English": 0.24431818181818182, "Vietnamese,Indonesian,Filipino,Chinese": 0.26704545454545453, "Vietnamese,Indonesian,English,Chinese": 0.26704545454545453, "Vietnamese,Spanish,Malay,Filipino": 0.1590909090909091, "Vietnamese,Spanish,Malay,English": 0.17045454545454544, "Vietnamese,Spanish,Malay,Chinese": 0.22727272727272727, "Vietnamese,Spanish,Filipino,English": 0.16477272727272727, "Vietnamese,Spanish,Filipino,Chinese": 0.17613636363636365, "Vietnamese,Spanish,English,Chinese": 0.19886363636363635, "Vietnamese,Malay,Filipino,English": 0.24431818181818182, "Vietnamese,Malay,Filipino,Chinese": 0.2727272727272727, "Vietnamese,Malay,English,Chinese": 0.26136363636363635, "Vietnamese,Filipino,English,Chinese": 0.26136363636363635, "Indonesian,Spanish,Malay,Filipino": 0.16477272727272727, "Indonesian,Spanish,Malay,English": 0.19886363636363635, "Indonesian,Spanish,Malay,Chinese": 0.19886363636363635, "Indonesian,Spanish,Filipino,English": 0.18181818181818182, "Indonesian,Spanish,Filipino,Chinese": 0.17045454545454544, "Indonesian,Spanish,English,Chinese": 0.19318181818181818, "Indonesian,Malay,Filipino,English": 0.23295454545454544, "Indonesian,Malay,Filipino,Chinese": 0.23863636363636365, "Indonesian,Malay,English,Chinese": 0.26704545454545453, "Indonesian,Filipino,English,Chinese": 0.26136363636363635, "Spanish,Malay,Filipino,English": 0.17045454545454544, "Spanish,Malay,Filipino,Chinese": 0.1875, "Spanish,Malay,English,Chinese": 0.1875, "Spanish,Filipino,English,Chinese": 0.17045454545454544, "Malay,Filipino,English,Chinese": 0.25 }, "5_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino": 0.11363636363636363, "Vietnamese,Indonesian,Spanish,Malay,English": 0.125, "Vietnamese,Indonesian,Spanish,Malay,Chinese": 0.1534090909090909, "Vietnamese,Indonesian,Spanish,Filipino,English": 0.13068181818181818, "Vietnamese,Indonesian,Spanish,Filipino,Chinese": 0.13068181818181818, "Vietnamese,Indonesian,Spanish,English,Chinese": 0.14772727272727273, "Vietnamese,Indonesian,Malay,Filipino,English": 0.17045454545454544, "Vietnamese,Indonesian,Malay,Filipino,Chinese": 0.18181818181818182, "Vietnamese,Indonesian,Malay,English,Chinese": 0.1875, "Vietnamese,Indonesian,Filipino,English,Chinese": 0.19886363636363635, "Vietnamese,Spanish,Malay,Filipino,English": 0.11931818181818182, "Vietnamese,Spanish,Malay,Filipino,Chinese": 0.13068181818181818, "Vietnamese,Spanish,Malay,English,Chinese": 0.13636363636363635, "Vietnamese,Spanish,Filipino,English,Chinese": 0.13068181818181818, "Vietnamese,Malay,Filipino,English,Chinese": 0.19318181818181818, "Indonesian,Spanish,Malay,Filipino,English": 0.13068181818181818, "Indonesian,Spanish,Malay,Filipino,Chinese": 0.11931818181818182, "Indonesian,Spanish,Malay,English,Chinese": 0.13636363636363635, "Indonesian,Spanish,Filipino,English,Chinese": 0.125, "Indonesian,Malay,Filipino,English,Chinese": 0.18181818181818182, "Spanish,Malay,Filipino,English,Chinese": 0.11931818181818182 }, "6_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English": 0.09090909090909091, "Vietnamese,Indonesian,Spanish,Malay,Filipino,Chinese": 0.09659090909090909, "Vietnamese,Indonesian,Spanish,Malay,English,Chinese": 0.10227272727272728, "Vietnamese,Indonesian,Spanish,Filipino,English,Chinese": 0.10227272727272728, "Vietnamese,Indonesian,Malay,Filipino,English,Chinese": 0.14204545454545456, "Vietnamese,Spanish,Malay,Filipino,English,Chinese": 0.09659090909090909, "Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.09090909090909091 }, "7_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.07386363636363637 } }, "AC3_2": 0.34702412120973286, "AC3_3": 0.28725146130568796, "AC3_4": 0.23234261400877593, "AC3_5": 0.1855165428301371, "AC3_6": 0.14679594777534632, "AC3_7": 0.11453423116605685 } }, "sg_eval": { "prompt_1": { "accuracy": 0.30097087378640774 } }, "cn_eval": { "prompt_1": { "accuracy": 0.26666666666666666 } }, "us_eval": { "prompt_1": { "accuracy": 0.2616822429906542 } }, "ph_eval": { "prompt_1": { "accuracy": 0.24, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.2, "history": 0.3333333333333333, "literature": 0.3, "politics": 0.2, "culture": 0.1, "film": 0.3, "law": 0.2, "geography": 0.3 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.2462549801651371 } }, "indommlu": { "prompt_1": { "accuracy": 0.24100407236798185, "category_acc": { "History": 0.2389558232931727, "Geography": 0.20816326530612245, "Lampungic": 0.29931972789115646, "Social science": 0.2904841402337229, "Balinese": 0.2229299363057325, "Makassarese": 0.25268817204301075, "Banjarese": 0.2708333333333333, "Chemistry": 0.2029197080291971, "Biology": 0.21301775147928995, "Science": 0.25077399380804954, "Christian religion": 0.1791044776119403, "Art": 0.22129783693843594, "Islam religion": 0.2361308677098151, "Hindu religion": 0.21333333333333335, "Madurese": 0.3152542372881356, "Sport": 0.20270270270270271, "Indonesian language": 0.24906600249066002, "Physics": 0.24444444444444444, "Minangkabau culture": 0.2914572864321608, "Dayak language": 0.1834862385321101, "Sociology": 0.21169354838709678, "Economy": 0.20491803278688525, "Sundanese": 0.2601555747623163, "Javanese": 0.24798387096774194, "Civic education": 0.2532188841201717 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.2670523221284094 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.22550154514283793 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.1515830576956539 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.2800385869875692 } }, "mmlu": { "prompt_1": { "accuracy": 0.26837806301050177 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.2664998212370397, "category_acc": { "high_school_european_history": 0.27439024390243905, "business_ethics": 0.29292929292929293, "clinical_knowledge": 0.25, "medical_genetics": 0.2222222222222222, "high_school_us_history": 0.33497536945812806, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.23728813559322035, "virology": 0.296969696969697, "high_school_microeconomics": 0.28270042194092826, "econometrics": 0.2920353982300885, "college_computer_science": 0.25252525252525254, "high_school_biology": 0.2588996763754045, "abstract_algebra": 0.24242424242424243, "professional_accounting": 0.27402135231316727, "philosophy": 0.27419354838709675, "professional_medicine": 0.25830258302583026, "nutrition": 0.26229508196721313, "global_facts": 0.26262626262626265, "machine_learning": 0.23423423423423423, "security_studies": 0.2336065573770492, "public_relations": 0.26605504587155965, "professional_psychology": 0.25040916530278234, "prehistory": 0.2631578947368421, "anatomy": 0.35074626865671643, "human_sexuality": 0.3230769230769231, "college_medicine": 0.27325581395348836, "high_school_government_and_politics": 0.296875, "college_chemistry": 0.24242424242424243, "logical_fallacies": 0.3333333333333333, "high_school_geography": 0.23857868020304568, "elementary_mathematics": 0.23342175066312998, "human_aging": 0.24324324324324326, "college_mathematics": 0.32323232323232326, "high_school_psychology": 0.2536764705882353, "formal_logic": 0.256, "high_school_statistics": 0.2837209302325581, "international_law": 0.2833333333333333, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.31313131313131315, "conceptual_physics": 0.2564102564102564, "miscellaneous": 0.2544757033248082, "high_school_chemistry": 0.28217821782178215, "marketing": 0.2446351931330472, "professional_law": 0.26614481409001955, "management": 0.3235294117647059, "college_physics": 0.21782178217821782, "jurisprudence": 0.2336448598130841, "world_religions": 0.18235294117647058, "sociology": 0.28, "us_foreign_policy": 0.2828282828282828, "high_school_macroeconomics": 0.2879177377892031, "computer_security": 0.24242424242424243, "moral_scenarios": 0.25838926174496646, "moral_disputes": 0.2985507246376812, "electrical_engineering": 0.25, "astronomy": 0.36423841059602646, "college_biology": 0.26573426573426573 } } }, "c_eval": { "prompt_1": { "accuracy": 0.25037147102526003 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.2602739726027397, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.16666666666666666, "computer_architecture": 0.2692307692307692, "college_programming": 0.2857142857142857, "college_physics": 0.25, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.08695652173913043, "high_school_physics": 0.375, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.08333333333333333, "middle_school_mathematics": 0.25, "middle_school_biology": 0.19230769230769232, "middle_school_physics": 0.25, "middle_school_chemistry": 0.12, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.21666666666666667, "business_administration": 0.3157894736842105, "marxism": 0.375, "mao_zedong_thought": 0.3103448275862069, "education_science": 0.3235294117647059, "teacher_qualification": 0.24489795918367346, "high_school_politics": 0.375, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.34615384615384615, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.3333333333333333, "law": 0.20689655172413793, "chinese_language_and_literature": 0.21428571428571427, "art_studies": 0.18421052631578946, "professional_tour_guide": 0.08823529411764706, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.25, "high_school_history": 0.32, "middle_school_history": 0.25925925925925924, "civil_servant": 0.25, "sports_science": 0.25, "plant_protection": 0.25925925925925924, "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.2549019607843137, "accountant": 0.2037037037037037, "fire_engineer": 0.3333333333333333, "environmental_impact_assessment_engineer": 0.3611111111111111, "tax_accountant": 0.2222222222222222, "physician": 0.2777777777777778 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.27598566308243727 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.2523743740286652, "category_acc": { "agronomy": 0.24260355029585798, "anatomy": 0.28378378378378377, "ancient_chinese": 0.23170731707317074, "arts": 0.3, "astronomy": 0.3393939393939394, "business_ethics": 0.27751196172248804, "chinese_civil_service_exam": 0.2375, "chinese_driving_rule": 0.25190839694656486, "chinese_food_culture": 0.22058823529411764, "chinese_foreign_policy": 0.2616822429906542, "chinese_history": 0.23839009287925697, "chinese_literature": 0.25980392156862747, "chinese_teacher_qualification": 0.2737430167597765, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.3113207547169811, "college_education": 0.2616822429906542, "college_engineering_hydrology": 0.1792452830188679, "college_law": 0.25, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.25471698113207547, "college_medicine": 0.2454212454212454, "computer_science": 0.27450980392156865, "computer_security": 0.23976608187134502, "conceptual_physics": 0.2789115646258503, "construction_project_management": 0.20863309352517986, "economics": 0.29559748427672955, "education": 0.22699386503067484, "electrical_engineering": 0.20930232558139536, "elementary_chinese": 0.27380952380952384, "elementary_commonsense": 0.22727272727272727, "elementary_information_and_technology": 0.22268907563025211, "elementary_mathematics": 0.24782608695652175, "ethnology": 0.2962962962962963, "food_science": 0.3006993006993007, "genetics": 0.2215909090909091, "global_facts": 0.24161073825503357, "high_school_biology": 0.23076923076923078, "high_school_chemistry": 0.25, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.25874125874125875, "human_sexuality": 0.3253968253968254, "international_law": 0.23783783783783785, "journalism": 0.21511627906976744, "jurisprudence": 0.24817518248175183, "legal_and_moral_basis": 0.29906542056074764, "logical": 0.2601626016260163, "machine_learning": 0.20491803278688525, "management": 0.2523809523809524, "marketing": 0.2777777777777778, "marxist_theory": 0.2328042328042328, "modern_chinese": 0.2413793103448276, "nutrition": 0.27586206896551724, "philosophy": 0.24761904761904763, "professional_accounting": 0.2342857142857143, "professional_law": 0.27014218009478674, "professional_medicine": 0.23404255319148937, "professional_psychology": 0.2543103448275862, "public_relations": 0.23563218390804597, "security_study": 0.25925925925925924, "sociology": 0.22566371681415928, "sports_science": 0.2545454545454545, "traditional_chinese_medicine": 0.21621621621621623, "virology": 0.25443786982248523, "world_history": 0.2608695652173913, "world_religions": 0.275 } } }, "zbench": { "prompt_1": { "accuracy": 0.21212121212121213 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.2 } }, "ocnli": { "prompt_1": { "accuracy": 0.3284745762711864 } }, "c3": { "prompt_1": { "accuracy": 0.30777860882572927 } }, "dream": { "prompt_1": { "accuracy": 0.3557079862812347 } }, "samsum": { "prompt_1": { "rouge1": 0.3750676640196728, "rouge2": 0.138430777796688, "rougeL": 0.3028717826203541, "avg_rouge": 0.2721234081455716 } }, "dialogsum": { "prompt_1": { "rouge1": 0.35512342177304645, "rouge2": 0.11827530044184112, "rougeL": 0.2910624826180367, "avg_rouge": 0.25482040161097474 } }, "sst2": { "prompt_1": { "accuracy": 0.5137614678899083 } }, "cola": { "prompt_1": { "accuracy": 0.5426653883029722 } }, "qqp": { "prompt_1": { "accuracy": 0.497 } }, "mnli": { "prompt_1": { "accuracy": 0.3 } }, "qnli": { "prompt_1": { "accuracy": 0.6 } }, "wnli": { "prompt_1": { "accuracy": 0.5 } }, "rte": { "prompt_1": { "accuracy": 0.5 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 } } } }, "sea_lion_7b": { "model_size": "7B", "model_link": "https://huggingface.co/aisingapore/sea-lion-7b", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.2647058823529412, "language_acc": { "Vietnamese": 0.2596638655462185, "Spanish": 0.2596638655462185, "English": 0.27899159663865547, "Chinese": 0.2605042016806723 }, "consistency_score_2": 0.34873949579831937, "consistency_score_3": 0.15399159663865547, "consistency_score_4": 0.07815126050420168, "detailed_consistency_score": { "2_combine": { "Vietnamese,Spanish": 0.3487394957983193, "Vietnamese,English": 0.37478991596638656, "Vietnamese,Chinese": 0.3764705882352941, "Spanish,English": 0.33613445378151263, "Spanish,Chinese": 0.3100840336134454, "English,Chinese": 0.346218487394958 }, "3_combine": { "Vietnamese,Spanish,English": 0.1546218487394958, "Vietnamese,Spanish,Chinese": 0.15126050420168066, "Vietnamese,English,Chinese": 0.173109243697479, "Spanish,English,Chinese": 0.1369747899159664 }, "4_combine": { "Vietnamese,Spanish,English,Chinese": 0.07815126050420168 } }, "AC3_2": 0.30096696207825496, "AC3_3": 0.19471089986790235, "AC3_4": 0.12067474044923225 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.28190476190476194, "language_acc": { "Chinese": 0.23333333333333334, "Indonesian": 0.24666666666666667, "Spanish": 0.28, "Vietnamese": 0.25333333333333335, "Malay": 0.38666666666666666, "English": 0.3333333333333333, "Filipino": 0.24 }, "consistency_score_2": 0.42793650793650784, "consistency_score_3": 0.23695238095238094, "consistency_score_4": 0.1500952380952381, "consistency_score_5": 0.10349206349206347, "consistency_score_6": 0.07619047619047618, "consistency_score_7": 0.06, "detailed_consistency_score": { "2_combine": { "Chinese,Indonesian": 0.43333333333333335, "Chinese,Spanish": 0.3333333333333333, "Chinese,Vietnamese": 0.44666666666666666, "Chinese,Malay": 0.4266666666666667, "Chinese,English": 0.4, "Chinese,Filipino": 0.4666666666666667, "Indonesian,Spanish": 0.36666666666666664, "Indonesian,Vietnamese": 0.44666666666666666, "Indonesian,Malay": 0.5133333333333333, "Indonesian,English": 0.42, "Indonesian,Filipino": 0.5, "Spanish,Vietnamese": 0.4, "Spanish,Malay": 0.4066666666666667, "Spanish,English": 0.4066666666666667, "Spanish,Filipino": 0.4066666666666667, "Vietnamese,Malay": 0.43333333333333335, "Vietnamese,English": 0.4533333333333333, "Vietnamese,Filipino": 0.46, "Malay,English": 0.38666666666666666, "Malay,Filipino": 0.4066666666666667, "English,Filipino": 0.47333333333333333 }, "3_combine": { "Chinese,Indonesian,Spanish": 0.16666666666666666, "Chinese,Indonesian,Vietnamese": 0.26, "Chinese,Indonesian,Malay": 0.25333333333333335, "Chinese,Indonesian,English": 0.22666666666666666, "Chinese,Indonesian,Filipino": 0.29333333333333333, "Chinese,Spanish,Vietnamese": 0.18, "Chinese,Spanish,Malay": 0.2, "Chinese,Spanish,English": 0.18, "Chinese,Spanish,Filipino": 0.18666666666666668, "Chinese,Vietnamese,Malay": 0.25333333333333335, "Chinese,Vietnamese,English": 0.23333333333333334, "Chinese,Vietnamese,Filipino": 0.25333333333333335, "Chinese,Malay,English": 0.21333333333333335, "Chinese,Malay,Filipino": 0.24666666666666667, "Chinese,English,Filipino": 0.26, "Indonesian,Spanish,Vietnamese": 0.22, "Indonesian,Spanish,Malay": 0.24, "Indonesian,Spanish,English": 0.20666666666666667, "Indonesian,Spanish,Filipino": 0.23333333333333334, "Indonesian,Vietnamese,Malay": 0.2733333333333333, "Indonesian,Vietnamese,English": 0.26666666666666666, "Indonesian,Vietnamese,Filipino": 0.3, "Indonesian,Malay,English": 0.26, "Indonesian,Malay,Filipino": 0.29333333333333333, "Indonesian,English,Filipino": 0.28, "Spanish,Vietnamese,Malay": 0.23333333333333334, "Spanish,Vietnamese,English": 0.21333333333333335, "Spanish,Vietnamese,Filipino": 0.22, "Spanish,Malay,English": 0.20666666666666667, "Spanish,Malay,Filipino": 0.21333333333333335, "Spanish,English,Filipino": 0.20666666666666667, "Vietnamese,Malay,English": 0.24666666666666667, "Vietnamese,Malay,Filipino": 0.26666666666666666, "Vietnamese,English,Filipino": 0.2866666666666667, "Malay,English,Filipino": 0.22 }, "4_combine": { "Chinese,Indonesian,Spanish,Vietnamese": 0.11333333333333333, "Chinese,Indonesian,Spanish,Malay": 0.12666666666666668, "Chinese,Indonesian,Spanish,English": 0.11333333333333333, "Chinese,Indonesian,Spanish,Filipino": 0.13333333333333333, "Chinese,Indonesian,Vietnamese,Malay": 0.18, "Chinese,Indonesian,Vietnamese,English": 0.16666666666666666, "Chinese,Indonesian,Vietnamese,Filipino": 0.18666666666666668, "Chinese,Indonesian,Malay,English": 0.15333333333333332, "Chinese,Indonesian,Malay,Filipino": 0.18, "Chinese,Indonesian,English,Filipino": 0.17333333333333334, "Chinese,Spanish,Vietnamese,Malay": 0.12666666666666668, "Chinese,Spanish,Vietnamese,English": 0.11333333333333333, "Chinese,Spanish,Vietnamese,Filipino": 0.12, "Chinese,Spanish,Malay,English": 0.12, "Chinese,Spanish,Malay,Filipino": 0.12666666666666668, "Chinese,Spanish,English,Filipino": 0.11333333333333333, "Chinese,Vietnamese,Malay,English": 0.15333333333333332, "Chinese,Vietnamese,Malay,Filipino": 0.17333333333333334, "Chinese,Vietnamese,English,Filipino": 0.18, "Chinese,Malay,English,Filipino": 0.14, "Indonesian,Spanish,Vietnamese,Malay": 0.15333333333333332, "Indonesian,Spanish,Vietnamese,English": 0.14, "Indonesian,Spanish,Vietnamese,Filipino": 0.15333333333333332, "Indonesian,Spanish,Malay,English": 0.14666666666666667, "Indonesian,Spanish,Malay,Filipino": 0.16, "Indonesian,Spanish,English,Filipino": 0.12, "Indonesian,Vietnamese,Malay,English": 0.18666666666666668, "Indonesian,Vietnamese,Malay,Filipino": 0.19333333333333333, "Indonesian,Vietnamese,English,Filipino": 0.2, "Indonesian,Malay,English,Filipino": 0.16666666666666666, "Spanish,Vietnamese,Malay,English": 0.14, "Spanish,Vietnamese,Malay,Filipino": 0.14666666666666667, "Spanish,Vietnamese,English,Filipino": 0.14, "Spanish,Malay,English,Filipino": 0.12, "Vietnamese,Malay,English,Filipino": 0.19333333333333333 }, "5_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay": 0.09333333333333334, "Chinese,Indonesian,Spanish,Vietnamese,English": 0.08666666666666667, "Chinese,Indonesian,Spanish,Vietnamese,Filipino": 0.09333333333333334, "Chinese,Indonesian,Spanish,Malay,English": 0.08666666666666667, "Chinese,Indonesian,Spanish,Malay,Filipino": 0.1, "Chinese,Indonesian,Spanish,English,Filipino": 0.08, "Chinese,Indonesian,Vietnamese,Malay,English": 0.12666666666666668, "Chinese,Indonesian,Vietnamese,Malay,Filipino": 0.13333333333333333, "Chinese,Indonesian,Vietnamese,English,Filipino": 0.13333333333333333, "Chinese,Indonesian,Malay,English,Filipino": 0.11333333333333333, "Chinese,Spanish,Vietnamese,Malay,English": 0.08666666666666667, "Chinese,Spanish,Vietnamese,Malay,Filipino": 0.09333333333333334, "Chinese,Spanish,Vietnamese,English,Filipino": 0.09333333333333334, "Chinese,Spanish,Malay,English,Filipino": 0.07333333333333333, "Chinese,Vietnamese,Malay,English,Filipino": 0.13333333333333333, "Indonesian,Spanish,Vietnamese,Malay,English": 0.10666666666666667, "Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.11333333333333333, "Indonesian,Spanish,Vietnamese,English,Filipino": 0.09333333333333334, "Indonesian,Spanish,Malay,English,Filipino": 0.08666666666666667, "Indonesian,Vietnamese,Malay,English,Filipino": 0.14666666666666667, "Spanish,Vietnamese,Malay,English,Filipino": 0.1 }, "6_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English": 0.07333333333333333, "Chinese,Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.08, "Chinese,Indonesian,Spanish,Vietnamese,English,Filipino": 0.06666666666666667, "Chinese,Indonesian,Spanish,Malay,English,Filipino": 0.06, "Chinese,Indonesian,Vietnamese,Malay,English,Filipino": 0.10666666666666667, "Chinese,Spanish,Vietnamese,Malay,English,Filipino": 0.07333333333333333, "Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.07333333333333333 }, "7_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.06 } }, "AC3_2": 0.3398994803167128, "AC3_3": 0.25748129496470995, "AC3_4": 0.19589149235408365, "AC3_5": 0.15140189844663354, "AC3_6": 0.11995947311746318, "AC3_7": 0.09894150414933466 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.273538961038961, "language_acc": { "Vietnamese": 0.3068181818181818, "Indonesian": 0.26704545454545453, "Spanish": 0.30113636363636365, "Malay": 0.2897727272727273, "Filipino": 0.23863636363636365, "English": 0.26136363636363635, "Chinese": 0.25 }, "consistency_score_2": 0.37878787878787873, "consistency_score_3": 0.17094155844155845, "consistency_score_4": 0.08409090909090906, "consistency_score_5": 0.04356060606060606, "consistency_score_6": 0.022727272727272728, "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { "Vietnamese,Indonesian": 0.38636363636363635, "Vietnamese,Spanish": 0.375, "Vietnamese,Malay": 0.4147727272727273, "Vietnamese,Filipino": 0.4431818181818182, "Vietnamese,English": 0.4147727272727273, "Vietnamese,Chinese": 0.42045454545454547, "Indonesian,Spanish": 0.36363636363636365, "Indonesian,Malay": 0.4318181818181818, "Indonesian,Filipino": 0.38636363636363635, "Indonesian,English": 0.3693181818181818, "Indonesian,Chinese": 0.36363636363636365, "Spanish,Malay": 0.26704545454545453, "Spanish,Filipino": 0.3693181818181818, "Spanish,English": 0.375, "Spanish,Chinese": 0.375, "Malay,Filipino": 0.35795454545454547, "Malay,English": 0.26704545454545453, "Malay,Chinese": 0.3522727272727273, "Filipino,English": 0.42613636363636365, "Filipino,Chinese": 0.3977272727272727, "English,Chinese": 0.3977272727272727 }, "3_combine": { "Vietnamese,Indonesian,Spanish": 0.17045454545454544, "Vietnamese,Indonesian,Malay": 0.21022727272727273, "Vietnamese,Indonesian,Filipino": 0.21022727272727273, "Vietnamese,Indonesian,English": 0.1590909090909091, "Vietnamese,Indonesian,Chinese": 0.18181818181818182, "Vietnamese,Spanish,Malay": 0.1534090909090909, "Vietnamese,Spanish,Filipino": 0.20454545454545456, "Vietnamese,Spanish,English": 0.1590909090909091, "Vietnamese,Spanish,Chinese": 0.17613636363636365, "Vietnamese,Malay,Filipino": 0.20454545454545456, "Vietnamese,Malay,English": 0.1534090909090909, "Vietnamese,Malay,Chinese": 0.19318181818181818, "Vietnamese,Filipino,English": 0.23863636363636365, "Vietnamese,Filipino,Chinese": 0.19886363636363635, "Vietnamese,English,Chinese": 0.1875, "Indonesian,Spanish,Malay": 0.14204545454545456, "Indonesian,Spanish,Filipino": 0.1590909090909091, "Indonesian,Spanish,English": 0.14772727272727273, "Indonesian,Spanish,Chinese": 0.1590909090909091, "Indonesian,Malay,Filipino": 0.19886363636363635, "Indonesian,Malay,English": 0.13068181818181818, "Indonesian,Malay,Chinese": 0.17613636363636365, "Indonesian,Filipino,English": 0.18181818181818182, "Indonesian,Filipino,Chinese": 0.18181818181818182, "Indonesian,English,Chinese": 0.17045454545454544, "Spanish,Malay,Filipino": 0.11931818181818182, "Spanish,Malay,English": 0.09090909090909091, "Spanish,Malay,Chinese": 0.14204545454545456, "Spanish,Filipino,English": 0.17045454545454544, "Spanish,Filipino,Chinese": 0.19318181818181818, "Spanish,English,Chinese": 0.17613636363636365, "Malay,Filipino,English": 0.14204545454545456, "Malay,Filipino,Chinese": 0.17045454545454544, "Malay,English,Chinese": 0.125, "Filipino,English,Chinese": 0.20454545454545456 }, "4_combine": { "Vietnamese,Indonesian,Spanish,Malay": 0.08522727272727272, "Vietnamese,Indonesian,Spanish,Filipino": 0.09090909090909091, "Vietnamese,Indonesian,Spanish,English": 0.0625, "Vietnamese,Indonesian,Spanish,Chinese": 0.09090909090909091, "Vietnamese,Indonesian,Malay,Filipino": 0.11931818181818182, "Vietnamese,Indonesian,Malay,English": 0.07954545454545454, "Vietnamese,Indonesian,Malay,Chinese": 0.09659090909090909, "Vietnamese,Indonesian,Filipino,English": 0.11931818181818182, "Vietnamese,Indonesian,Filipino,Chinese": 0.10227272727272728, "Vietnamese,Indonesian,English,Chinese": 0.09659090909090909, "Vietnamese,Spanish,Malay,Filipino": 0.07954545454545454, "Vietnamese,Spanish,Malay,English": 0.056818181818181816, "Vietnamese,Spanish,Malay,Chinese": 0.09090909090909091, "Vietnamese,Spanish,Filipino,English": 0.09659090909090909, "Vietnamese,Spanish,Filipino,Chinese": 0.08522727272727272, "Vietnamese,Spanish,English,Chinese": 0.09090909090909091, "Vietnamese,Malay,Filipino,English": 0.09090909090909091, "Vietnamese,Malay,Filipino,Chinese": 0.09659090909090909, "Vietnamese,Malay,English,Chinese": 0.07386363636363637, "Vietnamese,Filipino,English,Chinese": 0.11363636363636363, "Indonesian,Spanish,Malay,Filipino": 0.0625, "Indonesian,Spanish,Malay,English": 0.05113636363636364, "Indonesian,Spanish,Malay,Chinese": 0.07954545454545454, "Indonesian,Spanish,Filipino,English": 0.07954545454545454, "Indonesian,Spanish,Filipino,Chinese": 0.09090909090909091, "Indonesian,Spanish,English,Chinese": 0.07386363636363637, "Indonesian,Malay,Filipino,English": 0.07386363636363637, "Indonesian,Malay,Filipino,Chinese": 0.09659090909090909, "Indonesian,Malay,English,Chinese": 0.07386363636363637, "Indonesian,Filipino,English,Chinese": 0.10795454545454546, "Spanish,Malay,Filipino,English": 0.03977272727272727, "Spanish,Malay,Filipino,Chinese": 0.07386363636363637, "Spanish,Malay,English,Chinese": 0.05113636363636364, "Spanish,Filipino,English,Chinese": 0.09659090909090909, "Malay,Filipino,English,Chinese": 0.07386363636363637 }, "5_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino": 0.03409090909090909, "Vietnamese,Indonesian,Spanish,Malay,English": 0.028409090909090908, "Vietnamese,Indonesian,Spanish,Malay,Chinese": 0.056818181818181816, "Vietnamese,Indonesian,Spanish,Filipino,English": 0.045454545454545456, "Vietnamese,Indonesian,Spanish,Filipino,Chinese": 0.045454545454545456, "Vietnamese,Indonesian,Spanish,English,Chinese": 0.045454545454545456, "Vietnamese,Indonesian,Malay,Filipino,English": 0.056818181818181816, "Vietnamese,Indonesian,Malay,Filipino,Chinese": 0.05113636363636364, "Vietnamese,Indonesian,Malay,English,Chinese": 0.05113636363636364, "Vietnamese,Indonesian,Filipino,English,Chinese": 0.07386363636363637, "Vietnamese,Spanish,Malay,Filipino,English": 0.028409090909090908, "Vietnamese,Spanish,Malay,Filipino,Chinese": 0.03977272727272727, "Vietnamese,Spanish,Malay,English,Chinese": 0.03409090909090909, "Vietnamese,Spanish,Filipino,English,Chinese": 0.045454545454545456, "Vietnamese,Malay,Filipino,English,Chinese": 0.045454545454545456, "Indonesian,Spanish,Malay,Filipino,English": 0.022727272727272728, "Indonesian,Spanish,Malay,Filipino,Chinese": 0.045454545454545456, "Indonesian,Spanish,Malay,English,Chinese": 0.03977272727272727, "Indonesian,Spanish,Filipino,English,Chinese": 0.05113636363636364, "Indonesian,Malay,Filipino,English,Chinese": 0.05113636363636364, "Spanish,Malay,Filipino,English,Chinese": 0.022727272727272728 }, "6_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English": 0.011363636363636364, "Vietnamese,Indonesian,Spanish,Malay,Filipino,Chinese": 0.022727272727272728, "Vietnamese,Indonesian,Spanish,Malay,English,Chinese": 0.028409090909090908, "Vietnamese,Indonesian,Spanish,Filipino,English,Chinese": 0.028409090909090908, "Vietnamese,Indonesian,Malay,Filipino,English,Chinese": 0.03409090909090909, "Vietnamese,Spanish,Malay,Filipino,English,Chinese": 0.011363636363636364, "Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.022727272727272728 }, "7_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.011363636363636364 } }, "AC3_2": 0.317672787553579, "AC3_3": 0.21039921541944703, "AC3_4": 0.1286365699306053, "AC3_5": 0.07515319575662868, "AC3_6": 0.041967621405510705, "AC3_7": 0.02182077181311279 } }, "sg_eval": { "prompt_1": { "accuracy": 0.2621359223300971 } }, "cn_eval": { "prompt_1": { "accuracy": 0.23809523809523808 } }, "us_eval": { "prompt_1": { "accuracy": 0.205607476635514 } }, "ph_eval": { "prompt_1": { "accuracy": 0.3, "category_acc": { "brand": 0.1, "demographics": 0.4, "biology": 0.4, "history": 0.3333333333333333, "literature": 0.1, "politics": 0.2, "culture": 0.4, "film": 0.3, "law": 0.3, "geography": 0.5 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.24772930853092281 } }, "indommlu": { "prompt_1": { "accuracy": 0.24794712597636692, "category_acc": { "History": 0.22690763052208834, "Geography": 0.2, "Lampungic": 0.272108843537415, "Social science": 0.2687813021702838, "Balinese": 0.25477707006369427, "Makassarese": 0.24731182795698925, "Banjarese": 0.2361111111111111, "Chemistry": 0.19854014598540146, "Biology": 0.22011834319526627, "Science": 0.26418988648090813, "Christian religion": 0.3333333333333333, "Art": 0.2612312811980033, "Islam religion": 0.24039829302987198, "Hindu religion": 0.3333333333333333, "Madurese": 0.3050847457627119, "Sport": 0.21621621621621623, "Indonesian language": 0.2574719800747198, "Physics": 0.21616161616161617, "Minangkabau culture": 0.2914572864321608, "Dayak language": 0.24770642201834864, "Sociology": 0.17943548387096775, "Economy": 0.20491803278688525, "Sundanese": 0.29472774416594644, "Javanese": 0.2600806451612903, "Civic education": 0.21745350500715308 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.2794892349441752 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.24659996340969165 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.16575213960765156 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.2908730456568243 } }, "mmlu": { "prompt_1": { "accuracy": 0.24037339556592766 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.26642831605291384, "category_acc": { "high_school_european_history": 0.2621951219512195, "business_ethics": 0.26262626262626265, "clinical_knowledge": 0.2840909090909091, "medical_genetics": 0.26262626262626265, "high_school_us_history": 0.270935960591133, "high_school_physics": 0.3333333333333333, "high_school_world_history": 0.2796610169491525, "virology": 0.23636363636363636, "high_school_microeconomics": 0.2489451476793249, "econometrics": 0.22123893805309736, "college_computer_science": 0.43434343434343436, "high_school_biology": 0.2783171521035599, "abstract_algebra": 0.18181818181818182, "professional_accounting": 0.2669039145907473, "philosophy": 0.25806451612903225, "professional_medicine": 0.31365313653136534, "nutrition": 0.28524590163934427, "global_facts": 0.25252525252525254, "machine_learning": 0.2702702702702703, "security_studies": 0.2581967213114754, "public_relations": 0.23853211009174313, "professional_psychology": 0.25368248772504093, "prehistory": 0.26625386996904027, "anatomy": 0.291044776119403, "human_sexuality": 0.25384615384615383, "college_medicine": 0.27325581395348836, "high_school_government_and_politics": 0.2604166666666667, "college_chemistry": 0.2828282828282828, "logical_fallacies": 0.25925925925925924, "high_school_geography": 0.3604060913705584, "elementary_mathematics": 0.23342175066312998, "human_aging": 0.18468468468468469, "college_mathematics": 0.24242424242424243, "high_school_psychology": 0.2536764705882353, "formal_logic": 0.296, "high_school_statistics": 0.2744186046511628, "international_law": 0.3, "high_school_mathematics": 0.26022304832713755, "high_school_computer_science": 0.25252525252525254, "conceptual_physics": 0.2264957264957265, "miscellaneous": 0.24936061381074168, "high_school_chemistry": 0.2524752475247525, "marketing": 0.27896995708154504, "professional_law": 0.26810176125244617, "management": 0.21568627450980393, "college_physics": 0.21782178217821782, "jurisprudence": 0.308411214953271, "world_religions": 0.27058823529411763, "sociology": 0.275, "us_foreign_policy": 0.29292929292929293, "high_school_macroeconomics": 0.27249357326478146, "computer_security": 0.23232323232323232, "moral_scenarios": 0.2651006711409396, "moral_disputes": 0.30434782608695654, "electrical_engineering": 0.3055555555555556, "astronomy": 0.2847682119205298, "college_biology": 0.24475524475524477 } } }, "c_eval": { "prompt_1": { "accuracy": 0.26671619613670133 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.24719800747198006, "category_acc": { "computer_network": 0.375, "operating_system": 0.08333333333333333, "computer_architecture": 0.2692307692307692, "college_programming": 0.16666666666666666, "college_physics": 0.375, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.09523809523809523, "electrical_engineer": 0.23809523809523808, "metrology_engineer": 0.27586206896551724, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.16666666666666666, "high_school_biology": 0.25, "middle_school_mathematics": 0.4166666666666667, "middle_school_biology": 0.2692307692307692, "middle_school_physics": 0.375, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.21666666666666667, "business_administration": 0.2631578947368421, "marxism": 0.20833333333333334, "mao_zedong_thought": 0.3448275862068966, "education_science": 0.14705882352941177, "teacher_qualification": 0.22448979591836735, "high_school_politics": 0.20833333333333334, "high_school_geography": 0.041666666666666664, "middle_school_politics": 0.15384615384615385, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.25, "logic": 0.2222222222222222, "law": 0.2413793103448276, "chinese_language_and_literature": 0.10714285714285714, "art_studies": 0.2631578947368421, "professional_tour_guide": 0.20588235294117646, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.4583333333333333, "high_school_history": 0.44, "middle_school_history": 0.1111111111111111, "civil_servant": 0.28846153846153844, "sports_science": 0.4583333333333333, "plant_protection": 0.2962962962962963, "basic_medicine": 0.375, "clinical_medicine": 0.2222222222222222, "urban_and_rural_planner": 0.17647058823529413, "accountant": 0.25925925925925924, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.16666666666666666, "tax_accountant": 0.2222222222222222, "physician": 0.2222222222222222 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.25806451612903225 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.25470557762044554, "category_acc": { "agronomy": 0.20118343195266272, "anatomy": 0.2702702702702703, "ancient_chinese": 0.27439024390243905, "arts": 0.2125, "astronomy": 0.2787878787878788, "business_ethics": 0.2966507177033493, "chinese_civil_service_exam": 0.25625, "chinese_driving_rule": 0.22137404580152673, "chinese_food_culture": 0.2426470588235294, "chinese_foreign_policy": 0.18691588785046728, "chinese_history": 0.23839009287925697, "chinese_literature": 0.27450980392156865, "chinese_teacher_qualification": 0.2122905027932961, "clinical_knowledge": 0.2489451476793249, "college_actuarial_science": 0.2358490566037736, "college_education": 0.22429906542056074, "college_engineering_hydrology": 0.1792452830188679, "college_law": 0.24074074074074073, "college_mathematics": 0.1523809523809524, "college_medical_statistics": 0.27358490566037735, "college_medicine": 0.29304029304029305, "computer_science": 0.2647058823529412, "computer_security": 0.2807017543859649, "conceptual_physics": 0.22448979591836735, "construction_project_management": 0.26618705035971224, "economics": 0.2578616352201258, "education": 0.22699386503067484, "electrical_engineering": 0.3023255813953488, "elementary_chinese": 0.2777777777777778, "elementary_commonsense": 0.2777777777777778, "elementary_information_and_technology": 0.27310924369747897, "elementary_mathematics": 0.27391304347826084, "ethnology": 0.2814814814814815, "food_science": 0.27972027972027974, "genetics": 0.2784090909090909, "global_facts": 0.2550335570469799, "high_school_biology": 0.2781065088757396, "high_school_chemistry": 0.21212121212121213, "high_school_geography": 0.19491525423728814, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.19090909090909092, "high_school_politics": 0.21678321678321677, "human_sexuality": 0.30158730158730157, "international_law": 0.25405405405405407, "journalism": 0.26744186046511625, "jurisprudence": 0.27007299270072993, "legal_and_moral_basis": 0.2570093457943925, "logical": 0.2764227642276423, "machine_learning": 0.29508196721311475, "management": 0.2571428571428571, "marketing": 0.24444444444444444, "marxist_theory": 0.25396825396825395, "modern_chinese": 0.21551724137931033, "nutrition": 0.2827586206896552, "philosophy": 0.24761904761904763, "professional_accounting": 0.26857142857142857, "professional_law": 0.2890995260663507, "professional_medicine": 0.2393617021276596, "professional_psychology": 0.2629310344827586, "public_relations": 0.29310344827586204, "security_study": 0.23703703703703705, "sociology": 0.23893805309734514, "sports_science": 0.2787878787878788, "traditional_chinese_medicine": 0.22702702702702704, "virology": 0.23668639053254437, "world_history": 0.2546583850931677, "world_religions": 0.25 } } }, "zbench": { "prompt_1": { "accuracy": 0.21212121212121213 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.14772727272727273 } }, "ocnli": { "prompt_1": { "accuracy": 0.3183050847457627 } }, "c3": { "prompt_1": { "accuracy": 0.2931937172774869 } }, "dream": { "prompt_1": { "accuracy": 0.36354728074473297 } }, "samsum": { "prompt_1": { "rouge1": 0.39829958844759483, "rouge2": 0.15228600622383, "rougeL": 0.3226295820974558, "avg_rouge": 0.2910717255896269 } }, "dialogsum": { "prompt_1": { "rouge1": 0.3673257962967844, "rouge2": 0.1273838539423381, "rougeL": 0.3040291457636478, "avg_rouge": 0.26624626533425677 } }, "sst2": { "prompt_1": { "accuracy": 0.5206422018348624 } }, "cola": { "prompt_1": { "accuracy": 0.5647171620325983 } }, "qqp": { "prompt_1": { "accuracy": 0.496 } }, "mnli": { "prompt_1": { "accuracy": 0.5 } }, "qnli": { "prompt_1": { "accuracy": 0.5 } }, "wnli": { "prompt_1": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.8 } }, "mrpc": { "prompt_1": { "accuracy": 0.8 } } } }, "qwen1_5_110b": { "model_size": "110B", "model_link": "https://huggingface.co/Qwen/Qwen1.5-110B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.7514285714285714, "language_acc": { "Filipino": 0.74, "Vietnamese": 0.7266666666666667, "Chinese": 0.76, "Spanish": 0.78, "Malay": 0.7266666666666667, "Indonesian": 0.7533333333333333, "English": 0.7733333333333333 }, "consistency_score_2": 0.7933333333333333, "consistency_score_3": 0.7030476190476191, "consistency_score_4": 0.6436190476190474, "consistency_score_5": 0.598095238095238, "consistency_score_6": 0.5599999999999999, "consistency_score_7": 0.5266666666666666, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.7466666666666667, "Filipino,Chinese": 0.7533333333333333, "Filipino,Spanish": 0.7733333333333333, "Filipino,Malay": 0.72, "Filipino,Indonesian": 0.7466666666666667, "Filipino,English": 0.7733333333333333, "Vietnamese,Chinese": 0.7733333333333333, "Vietnamese,Spanish": 0.7866666666666666, "Vietnamese,Malay": 0.74, "Vietnamese,Indonesian": 0.8066666666666666, "Vietnamese,English": 0.8, "Chinese,Spanish": 0.84, "Chinese,Malay": 0.7666666666666667, "Chinese,Indonesian": 0.7933333333333333, "Chinese,English": 0.86, "Spanish,Malay": 0.7933333333333333, "Spanish,Indonesian": 0.86, "Spanish,English": 0.9066666666666666, "Malay,Indonesian": 0.8066666666666666, "Malay,English": 0.7866666666666666, "Indonesian,English": 0.8266666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.6533333333333333, "Filipino,Vietnamese,Spanish": 0.6733333333333333, "Filipino,Vietnamese,Malay": 0.62, "Filipino,Vietnamese,Indonesian": 0.6733333333333333, "Filipino,Vietnamese,English": 0.68, "Filipino,Chinese,Spanish": 0.7, "Filipino,Chinese,Malay": 0.6333333333333333, "Filipino,Chinese,Indonesian": 0.6666666666666666, "Filipino,Chinese,English": 0.7, "Filipino,Spanish,Malay": 0.6533333333333333, "Filipino,Spanish,Indonesian": 0.7066666666666667, "Filipino,Spanish,English": 0.7333333333333333, "Filipino,Malay,Indonesian": 0.64, "Filipino,Malay,English": 0.6466666666666666, "Filipino,Indonesian,English": 0.68, "Vietnamese,Chinese,Spanish": 0.7133333333333334, "Vietnamese,Chinese,Malay": 0.66, "Vietnamese,Chinese,Indonesian": 0.7, "Vietnamese,Chinese,English": 0.7266666666666667, "Vietnamese,Spanish,Malay": 0.68, "Vietnamese,Spanish,Indonesian": 0.74, "Vietnamese,Spanish,English": 0.76, "Vietnamese,Malay,Indonesian": 0.6866666666666666, "Vietnamese,Malay,English": 0.68, "Vietnamese,Indonesian,English": 0.7333333333333333, "Chinese,Spanish,Malay": 0.7133333333333334, "Chinese,Spanish,Indonesian": 0.76, "Chinese,Spanish,English": 0.8066666666666666, "Chinese,Malay,Indonesian": 0.6933333333333334, "Chinese,Malay,English": 0.72, "Chinese,Indonesian,English": 0.7533333333333333, "Spanish,Malay,Indonesian": 0.74, "Spanish,Malay,English": 0.7533333333333333, "Spanish,Indonesian,English": 0.8066666666666666, "Malay,Indonesian,English": 0.72 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.62, "Filipino,Vietnamese,Chinese,Malay": 0.5733333333333334, "Filipino,Vietnamese,Chinese,Indonesian": 0.6066666666666667, "Filipino,Vietnamese,Chinese,English": 0.6333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.5933333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.64, "Filipino,Vietnamese,Spanish,English": 0.6533333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.5866666666666667, "Filipino,Vietnamese,Malay,English": 0.5933333333333334, "Filipino,Vietnamese,Indonesian,English": 0.6333333333333333, "Filipino,Chinese,Spanish,Malay": 0.6066666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.6466666666666666, "Filipino,Chinese,Spanish,English": 0.68, "Filipino,Chinese,Malay,Indonesian": 0.5866666666666667, "Filipino,Chinese,Malay,English": 0.6, "Filipino,Chinese,Indonesian,English": 0.6333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.6133333333333333, "Filipino,Spanish,Malay,English": 0.6266666666666667, "Filipino,Spanish,Indonesian,English": 0.6666666666666666, "Filipino,Malay,Indonesian,English": 0.6, "Vietnamese,Chinese,Spanish,Malay": 0.64, "Vietnamese,Chinese,Spanish,Indonesian": 0.6733333333333333, "Vietnamese,Chinese,Spanish,English": 0.7, "Vietnamese,Chinese,Malay,Indonesian": 0.6266666666666667, "Vietnamese,Chinese,Malay,English": 0.64, "Vietnamese,Chinese,Indonesian,English": 0.6733333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.6533333333333333, "Vietnamese,Spanish,Malay,English": 0.6666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.72, "Vietnamese,Malay,Indonesian,English": 0.6533333333333333, "Chinese,Spanish,Malay,Indonesian": 0.6733333333333333, "Chinese,Spanish,Malay,English": 0.7, "Chinese,Spanish,Indonesian,English": 0.74, "Chinese,Malay,Indonesian,English": 0.6733333333333333, "Spanish,Malay,Indonesian,English": 0.7 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.56, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.5866666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.6133333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.5466666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.56, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.5866666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.5666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.58, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.62, "Filipino,Vietnamese,Malay,Indonesian,English": 0.5666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.5733333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.5933333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.6266666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.5666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.5866666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.6133333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.6333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.6666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.6133333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.64, "Chinese,Spanish,Malay,Indonesian,English": 0.66 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.5333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.5533333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.58, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.5333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.5533333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.56, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.6066666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.5266666666666666 } }, "AC3_2": 0.7718125770153882, "AC3_3": 0.7264334355558326, "AC3_4": 0.6933580400532867, "AC3_5": 0.6660510131576219, "AC3_6": 0.6417429193410435, "AC3_7": 0.6192846497279992 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.6607142857142857, "language_acc": { "Indonesian": 0.6420454545454546, "English": 0.75, "Filipino": 0.5227272727272727, "Spanish": 0.6818181818181818, "Chinese": 0.7954545454545454, "Malay": 0.6193181818181818, "Vietnamese": 0.6136363636363636 }, "consistency_score_2": 0.6988636363636364, "consistency_score_3": 0.572727272727273, "consistency_score_4": 0.4988636363636363, "consistency_score_5": 0.4483225108225109, "consistency_score_6": 0.40909090909090917, "consistency_score_7": 0.375, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7272727272727273, "Indonesian,Filipino": 0.6306818181818182, "Indonesian,Spanish": 0.7215909090909091, "Indonesian,Chinese": 0.6477272727272727, "Indonesian,Malay": 0.7784090909090909, "Indonesian,Vietnamese": 0.7045454545454546, "English,Filipino": 0.5909090909090909, "English,Spanish": 0.7954545454545454, "English,Chinese": 0.7329545454545454, "English,Malay": 0.6704545454545454, "English,Vietnamese": 0.6875, "Filipino,Spanish": 0.6761363636363636, "Filipino,Chinese": 0.6136363636363636, "Filipino,Malay": 0.6988636363636364, "Filipino,Vietnamese": 0.7102272727272727, "Spanish,Chinese": 0.7215909090909091, "Spanish,Malay": 0.7272727272727273, "Spanish,Vietnamese": 0.7727272727272727, "Chinese,Malay": 0.6363636363636364, "Chinese,Vietnamese": 0.6988636363636364, "Malay,Vietnamese": 0.7329545454545454 }, "3_combine": { "Indonesian,English,Filipino": 0.5170454545454546, "Indonesian,English,Spanish": 0.6420454545454546, "Indonesian,English,Chinese": 0.5795454545454546, "Indonesian,English,Malay": 0.6136363636363636, "Indonesian,English,Vietnamese": 0.5965909090909091, "Indonesian,Filipino,Spanish": 0.5454545454545454, "Indonesian,Filipino,Chinese": 0.48295454545454547, "Indonesian,Filipino,Malay": 0.5852272727272727, "Indonesian,Filipino,Vietnamese": 0.5568181818181818, "Indonesian,Spanish,Chinese": 0.5681818181818182, "Indonesian,Spanish,Malay": 0.6363636363636364, "Indonesian,Spanish,Vietnamese": 0.625, "Indonesian,Chinese,Malay": 0.5568181818181818, "Indonesian,Chinese,Vietnamese": 0.5568181818181818, "Indonesian,Malay,Vietnamese": 0.625, "English,Filipino,Spanish": 0.5568181818181818, "English,Filipino,Chinese": 0.48295454545454547, "English,Filipino,Malay": 0.5170454545454546, "English,Filipino,Vietnamese": 0.5284090909090909, "English,Spanish,Chinese": 0.6363636363636364, "English,Spanish,Malay": 0.6079545454545454, "English,Spanish,Vietnamese": 0.6477272727272727, "English,Chinese,Malay": 0.5397727272727273, "English,Chinese,Vietnamese": 0.5738636363636364, "English,Malay,Vietnamese": 0.5681818181818182, "Filipino,Spanish,Chinese": 0.5284090909090909, "Filipino,Spanish,Malay": 0.5738636363636364, "Filipino,Spanish,Vietnamese": 0.6022727272727273, "Filipino,Chinese,Malay": 0.5113636363636364, "Filipino,Chinese,Vietnamese": 0.5454545454545454, "Filipino,Malay,Vietnamese": 0.5852272727272727, "Spanish,Chinese,Malay": 0.5568181818181818, "Spanish,Chinese,Vietnamese": 0.6136363636363636, "Spanish,Malay,Vietnamese": 0.625, "Chinese,Malay,Vietnamese": 0.5568181818181818 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.48863636363636365, "Indonesian,English,Filipino,Chinese": 0.4318181818181818, "Indonesian,English,Filipino,Malay": 0.4772727272727273, "Indonesian,English,Filipino,Vietnamese": 0.4772727272727273, "Indonesian,English,Spanish,Chinese": 0.5170454545454546, "Indonesian,English,Spanish,Malay": 0.5681818181818182, "Indonesian,English,Spanish,Vietnamese": 0.5625, "Indonesian,English,Chinese,Malay": 0.5056818181818182, "Indonesian,English,Chinese,Vietnamese": 0.5, "Indonesian,English,Malay,Vietnamese": 0.5397727272727273, "Indonesian,Filipino,Spanish,Chinese": 0.4602272727272727, "Indonesian,Filipino,Spanish,Malay": 0.5170454545454546, "Indonesian,Filipino,Spanish,Vietnamese": 0.5056818181818182, "Indonesian,Filipino,Chinese,Malay": 0.45454545454545453, "Indonesian,Filipino,Chinese,Vietnamese": 0.45454545454545453, "Indonesian,Filipino,Malay,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Chinese,Malay": 0.5056818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.5170454545454546, "Indonesian,Spanish,Malay,Vietnamese": 0.5738636363636364, "Indonesian,Chinese,Malay,Vietnamese": 0.5056818181818182, "English,Filipino,Spanish,Chinese": 0.4715909090909091, "English,Filipino,Spanish,Malay": 0.4943181818181818, "English,Filipino,Spanish,Vietnamese": 0.5113636363636364, "English,Filipino,Chinese,Malay": 0.4318181818181818, "English,Filipino,Chinese,Vietnamese": 0.44886363636363635, "English,Filipino,Malay,Vietnamese": 0.4772727272727273, "English,Spanish,Chinese,Malay": 0.5, "English,Spanish,Chinese,Vietnamese": 0.5454545454545454, "English,Spanish,Malay,Vietnamese": 0.5511363636363636, "English,Chinese,Malay,Vietnamese": 0.48295454545454547, "Filipino,Spanish,Chinese,Malay": 0.4659090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.48863636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.5227272727272727, "Filipino,Chinese,Malay,Vietnamese": 0.4715909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.5113636363636364 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.42045454545454547, "Indonesian,English,Filipino,Spanish,Malay": 0.4602272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.4602272727272727, "Indonesian,English,Filipino,Chinese,Malay": 0.4034090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,English,Filipino,Malay,Vietnamese": 0.44886363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.4659090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4715909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5227272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4602272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4318181818181818, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4318181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.48295454545454547, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.4318181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4772727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4375, "English,Filipino,Spanish,Malay,Vietnamese": 0.4659090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.4090909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.4659090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4375 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.39204545454545453, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.3977272727272727, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4375, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.38636363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4431818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375 } }, "AC3_2": 0.679253731293323, "AC3_3": 0.6135825216664956, "AC3_4": 0.5684936300783714, "AC3_5": 0.5341808106025152, "AC3_6": 0.505311077342751, "AC3_7": 0.478448275815874 } }, "sg_eval": { "prompt_1": { "accuracy": 0.7281553398058253 } }, "cn_eval": { "prompt_1": { "accuracy": 0.9047619047619048 } }, "us_eval": { "prompt_1": { "accuracy": 0.8598130841121495 } }, "ph_eval": { "prompt_1": { "accuracy": 0.67, "category_acc": { "brand": 0.4, "demographics": 0.0, "biology": 0.7, "history": 0.6666666666666666, "literature": 0.6, "politics": 0.9, "culture": 1.0, "film": 0.6, "law": 0.9, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.3171697011113447 } }, "indommlu": { "prompt_1": { "accuracy": 0.6228720208291608, "category_acc": { "History": 0.6606425702811245, "Geography": 0.6530612244897959, "Lampungic": 0.3945578231292517, "Social science": 0.8547579298831386, "Balinese": 0.3609341825902335, "Makassarese": 0.46236559139784944, "Banjarese": 0.5277777777777778, "Chemistry": 0.4481751824817518, "Biology": 0.6437869822485207, "Science": 0.7956656346749226, "Christian religion": 0.7960199004975125, "Art": 0.7104825291181365, "Islam religion": 0.7453769559032717, "Hindu religion": 0.6333333333333333, "Madurese": 0.3288135593220339, "Sport": 0.5945945945945946, "Indonesian language": 0.6936488169364882, "Physics": 0.6080808080808081, "Minangkabau culture": 0.4723618090452261, "Dayak language": 0.3577981651376147, "Sociology": 0.6189516129032258, "Economy": 0.6598360655737705, "Sundanese": 0.4736387208297321, "Javanese": 0.4344758064516129, "Civic education": 0.709585121602289 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.34831928857399685 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.2884044318979437 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.25950573635717444 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.36918131956164535 } }, "mmlu": { "prompt_1": { "accuracy": 0.7689614935822637 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.7746156596353235, "category_acc": { "high_school_european_history": 0.8170731707317073, "business_ethics": 0.797979797979798, "clinical_knowledge": 0.8257575757575758, "medical_genetics": 0.8585858585858586, "high_school_us_history": 0.9310344827586207, "high_school_physics": 0.56, "high_school_world_history": 0.9152542372881356, "virology": 0.5575757575757576, "high_school_microeconomics": 0.8945147679324894, "econometrics": 0.5929203539823009, "college_computer_science": 0.6565656565656566, "high_school_biology": 0.9029126213592233, "abstract_algebra": 0.5151515151515151, "professional_accounting": 0.6548042704626335, "philosophy": 0.8290322580645161, "professional_medicine": 0.8856088560885609, "nutrition": 0.8590163934426229, "global_facts": 0.5555555555555556, "machine_learning": 0.6396396396396397, "security_studies": 0.7909836065573771, "public_relations": 0.7614678899082569, "professional_psychology": 0.8346972176759411, "prehistory": 0.891640866873065, "anatomy": 0.746268656716418, "human_sexuality": 0.8769230769230769, "college_medicine": 0.7441860465116279, "high_school_government_and_politics": 0.9895833333333334, "college_chemistry": 0.5757575757575758, "logical_fallacies": 0.8518518518518519, "high_school_geography": 0.9137055837563451, "elementary_mathematics": 0.6896551724137931, "human_aging": 0.8198198198198198, "college_mathematics": 0.5757575757575758, "high_school_psychology": 0.9264705882352942, "formal_logic": 0.584, "high_school_statistics": 0.7441860465116279, "international_law": 0.875, "high_school_mathematics": 0.5613382899628253, "high_school_computer_science": 0.8282828282828283, "conceptual_physics": 0.7948717948717948, "miscellaneous": 0.9335038363171355, "high_school_chemistry": 0.7425742574257426, "marketing": 0.9184549356223176, "professional_law": 0.6281800391389433, "management": 0.8627450980392157, "college_physics": 0.5247524752475248, "jurisprudence": 0.822429906542056, "world_religions": 0.8764705882352941, "sociology": 0.935, "us_foreign_policy": 0.9494949494949495, "high_school_macroeconomics": 0.8046272493573264, "computer_security": 0.8383838383838383, "moral_scenarios": 0.5447427293064877, "moral_disputes": 0.8115942028985508, "electrical_engineering": 0.7083333333333334, "astronomy": 0.9072847682119205, "college_biology": 0.9370629370629371 } } }, "c_eval": { "prompt_1": { "accuracy": 0.8692421991084696 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.8785803237858032, "category_acc": { "computer_network": 0.7916666666666666, "operating_system": 0.8333333333333334, "computer_architecture": 0.8461538461538461, "college_programming": 0.8333333333333334, "college_physics": 0.875, "college_chemistry": 0.7931034482758621, "advanced_mathematics": 0.5833333333333334, "probability_and_statistics": 0.5652173913043478, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.7857142857142857, "metrology_engineer": 0.9310344827586207, "high_school_mathematics": 0.6956521739130435, "high_school_physics": 0.9166666666666666, "high_school_chemistry": 0.9583333333333334, "high_school_biology": 1.0, "middle_school_mathematics": 0.9583333333333334, "middle_school_biology": 0.9230769230769231, "middle_school_physics": 1.0, "middle_school_chemistry": 1.0, "veterinary_medicine": 0.8571428571428571, "college_economics": 0.7, "business_administration": 0.8421052631578947, "marxism": 0.9583333333333334, "mao_zedong_thought": 1.0, "education_science": 0.9117647058823529, "teacher_qualification": 0.9591836734693877, "high_school_politics": 0.9583333333333334, "high_school_geography": 0.9166666666666666, "middle_school_politics": 0.9230769230769231, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.9285714285714286, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.8888888888888888, "law": 0.896551724137931, "chinese_language_and_literature": 0.7857142857142857, "art_studies": 0.9210526315789473, "professional_tour_guide": 0.9705882352941176, "legal_professional": 0.8571428571428571, "high_school_chinese": 0.875, "high_school_history": 1.0, "middle_school_history": 0.9629629629629629, "civil_servant": 0.9230769230769231, "sports_science": 0.9166666666666666, "plant_protection": 0.9629629629629629, "basic_medicine": 0.9166666666666666, "clinical_medicine": 0.9629629629629629, "urban_and_rural_planner": 0.8627450980392157, "accountant": 0.9444444444444444, "fire_engineer": 0.8888888888888888, "environmental_impact_assessment_engineer": 0.8611111111111112, "tax_accountant": 0.8703703703703703, "physician": 0.8888888888888888 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.8673835125448028 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.8836988430322915, "category_acc": { "agronomy": 0.8520710059171598, "anatomy": 0.9121621621621622, "ancient_chinese": 0.6646341463414634, "arts": 0.9875, "astronomy": 0.6484848484848484, "business_ethics": 0.7033492822966507, "chinese_civil_service_exam": 0.8875, "chinese_driving_rule": 1.0, "chinese_food_culture": 0.8970588235294118, "chinese_foreign_policy": 0.9813084112149533, "chinese_history": 0.9721362229102167, "chinese_literature": 0.8676470588235294, "chinese_teacher_qualification": 0.9497206703910615, "clinical_knowledge": 0.9029535864978903, "college_actuarial_science": 0.4056603773584906, "college_education": 0.9719626168224299, "college_engineering_hydrology": 0.9528301886792453, "college_law": 0.9259259259259259, "college_mathematics": 0.5523809523809524, "college_medical_statistics": 0.8490566037735849, "college_medicine": 0.945054945054945, "computer_science": 0.9215686274509803, "computer_security": 0.9532163742690059, "conceptual_physics": 0.9387755102040817, "construction_project_management": 0.8489208633093526, "economics": 0.9245283018867925, "education": 0.8343558282208589, "electrical_engineering": 0.9011627906976745, "elementary_chinese": 0.8611111111111112, "elementary_commonsense": 0.8686868686868687, "elementary_information_and_technology": 0.9453781512605042, "elementary_mathematics": 0.717391304347826, "ethnology": 0.9481481481481482, "food_science": 0.7692307692307693, "genetics": 0.7784090909090909, "global_facts": 0.912751677852349, "high_school_biology": 0.9763313609467456, "high_school_chemistry": 0.9242424242424242, "high_school_geography": 0.9745762711864406, "high_school_mathematics": 0.75, "high_school_physics": 0.8545454545454545, "high_school_politics": 0.9090909090909091, "human_sexuality": 0.8412698412698413, "international_law": 0.8540540540540541, "journalism": 0.8430232558139535, "jurisprudence": 0.9391727493917275, "legal_and_moral_basis": 0.985981308411215, "logical": 0.9186991869918699, "machine_learning": 0.819672131147541, "management": 0.9285714285714286, "marketing": 0.9444444444444444, "marxist_theory": 0.9735449735449735, "modern_chinese": 0.7931034482758621, "nutrition": 0.8758620689655172, "philosophy": 0.9142857142857143, "professional_accounting": 0.9657142857142857, "professional_law": 0.8388625592417062, "professional_medicine": 0.875, "professional_psychology": 0.9137931034482759, "public_relations": 0.8103448275862069, "security_study": 0.9481481481481482, "sociology": 0.9203539823008849, "sports_science": 0.9030303030303031, "traditional_chinese_medicine": 0.9351351351351351, "virology": 0.9289940828402367, "world_history": 0.968944099378882, "world_religions": 0.875 } } }, "zbench": { "prompt_1": { "accuracy": 0.5454545454545454 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6795454545454546 } }, "ocnli": { "prompt_1": { "accuracy": 0.7969491525423729 } }, "c3": { "prompt_1": { "accuracy": 0.9599850411368736 } }, "dream": { "prompt_1": { "accuracy": 0.9549240568348849 } }, "samsum": { "prompt_1": { "rouge1": 0.4431616176140729, "rouge2": 0.19487271543391588, "rougeL": 0.35620784890484203, "avg_rouge": 0.3314140606509436 } }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": { "accuracy": 0.9610091743119266 } }, "cola": { "prompt_1": { "accuracy": 0.8456375838926175 } }, "qqp": { "prompt_1": { "accuracy": 0.8315 } }, "mnli": { "prompt_1": { "accuracy": 0.6 } }, "qnli": { "prompt_1": { "accuracy": 0.7 } }, "wnli": { "prompt_1": { "accuracy": 0.6 } }, "rte": { "prompt_1": { "accuracy": 0.7 } }, "mrpc": { "prompt_1": { "accuracy": 0.7 } } } }, "qwen1_5_110b_chat": { "model_size": "110B", "model_link": "https://huggingface.co/Qwen/Qwen1.5-110B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": { "overall_acc": 0.9548319327731092, "language_acc": { "Spanish": 0.9537815126050421, "English": 0.9588235294117647, "Chinese": 0.9554621848739496, "Vietnamese": 0.9512605042016806 }, "consistency_score_2": 0.9519607843137255, "consistency_score_3": 0.9310924369747899, "consistency_score_4": 0.9193277310924369, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9613445378151261, "Spanish,Chinese": 0.9478991596638655, "Spanish,Vietnamese": 0.9445378151260504, "English,Chinese": 0.9537815126050421, "English,Vietnamese": 0.9554621848739496, "Chinese,Vietnamese": 0.9487394957983193 }, "3_combine": { "Spanish,English,Chinese": 0.9352941176470588, "Spanish,English,Vietnamese": 0.9336134453781513, "Spanish,Chinese,Vietnamese": 0.9243697478991597, "English,Chinese,Vietnamese": 0.9310924369747899 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9193277310924369 } }, "AC3_2": 0.9533941968810403, "AC3_3": 0.942812771711269, "AC3_4": 0.9367435349382761 }, "prompt_3": { "overall_acc": 0.9502100840336134, "language_acc": { "Spanish": 0.9478991596638655, "English": 0.9546218487394958, "Chinese": 0.9487394957983193, "Vietnamese": 0.9495798319327731 }, "consistency_score_2": 0.9449579831932774, "consistency_score_3": 0.9203781512605042, "consistency_score_4": 0.9033613445378151, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9495798319327731, "Spanish,Chinese": 0.9453781512605042, "Spanish,Vietnamese": 0.9411764705882353, "English,Chinese": 0.9478991596638655, "English,Vietnamese": 0.9436974789915966, "Chinese,Vietnamese": 0.9420168067226891 }, "3_combine": { "Spanish,English,Chinese": 0.9243697478991597, "Spanish,English,Vietnamese": 0.9201680672268907, "Spanish,Chinese,Vietnamese": 0.9176470588235294, "English,Chinese,Vietnamese": 0.9193277310924369 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9033613445378151 } }, "AC3_2": 0.9475767559600201, "AC3_3": 0.9350562394269354, "AC3_4": 0.9261936668081386 }, "prompt_4": { "overall_acc": 0.9552521008403362, "language_acc": { "Spanish": 0.9546218487394958, "English": 0.9571428571428572, "Chinese": 0.9546218487394958, "Vietnamese": 0.9546218487394958 }, "consistency_score_2": 0.9528011204481793, "consistency_score_3": 0.9323529411764706, "consistency_score_4": 0.9193277310924369, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9596638655462185, "Spanish,Chinese": 0.9512605042016806, "Spanish,Vietnamese": 0.9512605042016806, "English,Chinese": 0.9512605042016806, "English,Vietnamese": 0.9512605042016806, "Chinese,Vietnamese": 0.9521008403361344 }, "3_combine": { "Spanish,English,Chinese": 0.9352941176470588, "Spanish,English,Vietnamese": 0.9336134453781513, "Spanish,Chinese,Vietnamese": 0.9294117647058824, "English,Chinese,Vietnamese": 0.9310924369747899 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.9193277310924369 } }, "AC3_2": 0.9540250363968467, "AC3_3": 0.943663622327172, "AC3_4": 0.9369456893543243 }, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.680952380952381, "language_acc": { "Filipino": 0.6466666666666666, "Vietnamese": 0.64, "Chinese": 0.72, "Spanish": 0.7266666666666667, "Malay": 0.6466666666666666, "Indonesian": 0.6666666666666666, "English": 0.72 }, "consistency_score_2": 0.7130158730158731, "consistency_score_3": 0.6093333333333334, "consistency_score_4": 0.551047619047619, "consistency_score_5": 0.50984126984127, "consistency_score_6": 0.4761904761904762, "consistency_score_7": 0.44666666666666666, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.66, "Filipino,Chinese": 0.68, "Filipino,Spanish": 0.6733333333333333, "Filipino,Malay": 0.64, "Filipino,Indonesian": 0.6466666666666666, "Filipino,English": 0.6866666666666666, "Vietnamese,Chinese": 0.7266666666666667, "Vietnamese,Spanish": 0.7066666666666667, "Vietnamese,Malay": 0.7, "Vietnamese,Indonesian": 0.6933333333333334, "Vietnamese,English": 0.7, "Chinese,Spanish": 0.78, "Chinese,Malay": 0.68, "Chinese,Indonesian": 0.7333333333333333, "Chinese,English": 0.8, "Spanish,Malay": 0.72, "Spanish,Indonesian": 0.7533333333333333, "Spanish,English": 0.8266666666666667, "Malay,Indonesian": 0.72, "Malay,English": 0.7133333333333334, "Indonesian,English": 0.7333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5733333333333334, "Filipino,Vietnamese,Spanish": 0.5733333333333334, "Filipino,Vietnamese,Malay": 0.5533333333333333, "Filipino,Vietnamese,Indonesian": 0.5466666666666666, "Filipino,Vietnamese,English": 0.5733333333333334, "Filipino,Chinese,Spanish": 0.6, "Filipino,Chinese,Malay": 0.5533333333333333, "Filipino,Chinese,Indonesian": 0.56, "Filipino,Chinese,English": 0.6133333333333333, "Filipino,Spanish,Malay": 0.56, "Filipino,Spanish,Indonesian": 0.58, "Filipino,Spanish,English": 0.6333333333333333, "Filipino,Malay,Indonesian": 0.5533333333333333, "Filipino,Malay,English": 0.5666666666666667, "Filipino,Indonesian,English": 0.58, "Vietnamese,Chinese,Spanish": 0.64, "Vietnamese,Chinese,Malay": 0.5866666666666667, "Vietnamese,Chinese,Indonesian": 0.6, "Vietnamese,Chinese,English": 0.66, "Vietnamese,Spanish,Malay": 0.6133333333333333, "Vietnamese,Spanish,Indonesian": 0.6133333333333333, "Vietnamese,Spanish,English": 0.6466666666666666, "Vietnamese,Malay,Indonesian": 0.5866666666666667, "Vietnamese,Malay,English": 0.6, "Vietnamese,Indonesian,English": 0.6066666666666667, "Chinese,Spanish,Malay": 0.6266666666666667, "Chinese,Spanish,Indonesian": 0.6666666666666666, "Chinese,Spanish,English": 0.7266666666666667, "Chinese,Malay,Indonesian": 0.6, "Chinese,Malay,English": 0.6466666666666666, "Chinese,Indonesian,English": 0.6666666666666666, "Spanish,Malay,Indonesian": 0.6266666666666667, "Spanish,Malay,English": 0.6733333333333333, "Spanish,Indonesian,English": 0.68, "Malay,Indonesian,English": 0.64 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.54, "Filipino,Vietnamese,Chinese,Malay": 0.5, "Filipino,Vietnamese,Chinese,Indonesian": 0.5, "Filipino,Vietnamese,Chinese,English": 0.54, "Filipino,Vietnamese,Spanish,Malay": 0.5, "Filipino,Vietnamese,Spanish,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,Spanish,English": 0.5533333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.49333333333333335, "Filipino,Vietnamese,Malay,English": 0.5133333333333333, "Filipino,Vietnamese,Indonesian,English": 0.52, "Filipino,Chinese,Spanish,Malay": 0.52, "Filipino,Chinese,Spanish,Indonesian": 0.54, "Filipino,Chinese,Spanish,English": 0.5866666666666667, "Filipino,Chinese,Malay,Indonesian": 0.5066666666666667, "Filipino,Chinese,Malay,English": 0.5266666666666666, "Filipino,Chinese,Indonesian,English": 0.54, "Filipino,Spanish,Malay,Indonesian": 0.5133333333333333, "Filipino,Spanish,Malay,English": 0.5533333333333333, "Filipino,Spanish,Indonesian,English": 0.5666666666666667, "Filipino,Malay,Indonesian,English": 0.5266666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.56, "Vietnamese,Chinese,Spanish,Indonesian": 0.5666666666666667, "Vietnamese,Chinese,Spanish,English": 0.62, "Vietnamese,Chinese,Malay,Indonesian": 0.5333333333333333, "Vietnamese,Chinese,Malay,English": 0.5733333333333334, "Vietnamese,Chinese,Indonesian,English": 0.5733333333333334, "Vietnamese,Spanish,Malay,Indonesian": 0.56, "Vietnamese,Spanish,Malay,English": 0.58, "Vietnamese,Spanish,Indonesian,English": 0.5866666666666667, "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay,Indonesian": 0.5733333333333334, "Chinese,Spanish,Malay,English": 0.6266666666666667, "Chinese,Spanish,Indonesian,English": 0.6333333333333333, "Chinese,Malay,Indonesian,English": 0.5866666666666667, "Spanish,Malay,Indonesian,English": 0.6066666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.48, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4866666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.5266666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.46, "Filipino,Vietnamese,Chinese,Malay,English": 0.4866666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4866666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.5, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5066666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.48, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.4866666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.52, "Filipino,Chinese,Spanish,Indonesian,English": 0.5333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.49333333333333335, "Filipino,Spanish,Malay,Indonesian,English": 0.5133333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.52, "Vietnamese,Chinese,Spanish,Malay,English": 0.56, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.56, "Vietnamese,Chinese,Malay,Indonesian,English": 0.5266666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.54, "Chinese,Spanish,Malay,Indonesian,English": 0.5733333333333334 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.48, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.48, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.4533333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.4666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.4866666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.52 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 } }, "AC3_2": 0.6966153712180965, "AC3_3": 0.6431552012098726, "AC3_4": 0.6091512795099987, "AC3_5": 0.5831029185378221, "AC3_6": 0.5604546344802743, "AC3_7": 0.5394707206728793 }, "prompt_2": { "overall_acc": 0.660952380952381, "language_acc": { "Filipino": 0.6466666666666666, "Vietnamese": 0.64, "Chinese": 0.6733333333333333, "Spanish": 0.7, "Malay": 0.58, "Indonesian": 0.66, "English": 0.7266666666666667 }, "consistency_score_2": 0.6752380952380954, "consistency_score_3": 0.5586666666666668, "consistency_score_4": 0.49676190476190474, "consistency_score_5": 0.4555555555555556, "consistency_score_6": 0.42476190476190473, "consistency_score_7": 0.4, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6333333333333333, "Filipino,Chinese": 0.6733333333333333, "Filipino,Spanish": 0.68, "Filipino,Malay": 0.62, "Filipino,Indonesian": 0.6533333333333333, "Filipino,English": 0.6666666666666666, "Vietnamese,Chinese": 0.6933333333333334, "Vietnamese,Spanish": 0.6933333333333334, "Vietnamese,Malay": 0.6, "Vietnamese,Indonesian": 0.64, "Vietnamese,English": 0.7, "Chinese,Spanish": 0.7333333333333333, "Chinese,Malay": 0.6066666666666667, "Chinese,Indonesian": 0.7266666666666667, "Chinese,English": 0.7666666666666667, "Spanish,Malay": 0.6266666666666667, "Spanish,Indonesian": 0.7533333333333333, "Spanish,English": 0.76, "Malay,Indonesian": 0.66, "Malay,English": 0.6066666666666667, "Indonesian,English": 0.6866666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.56, "Filipino,Vietnamese,Spanish": 0.5466666666666666, "Filipino,Vietnamese,Malay": 0.48, "Filipino,Vietnamese,Indonesian": 0.5333333333333333, "Filipino,Vietnamese,English": 0.5466666666666666, "Filipino,Chinese,Spanish": 0.5866666666666667, "Filipino,Chinese,Malay": 0.5, "Filipino,Chinese,Indonesian": 0.5666666666666667, "Filipino,Chinese,English": 0.5933333333333334, "Filipino,Spanish,Malay": 0.5, "Filipino,Spanish,Indonesian": 0.58, "Filipino,Spanish,English": 0.58, "Filipino,Malay,Indonesian": 0.5133333333333333, "Filipino,Malay,English": 0.4866666666666667, "Filipino,Indonesian,English": 0.5533333333333333, "Vietnamese,Chinese,Spanish": 0.6, "Vietnamese,Chinese,Malay": 0.5133333333333333, "Vietnamese,Chinese,Indonesian": 0.58, "Vietnamese,Chinese,English": 0.62, "Vietnamese,Spanish,Malay": 0.5133333333333333, "Vietnamese,Spanish,Indonesian": 0.5866666666666667, "Vietnamese,Spanish,English": 0.6066666666666667, "Vietnamese,Malay,Indonesian": 0.5133333333333333, "Vietnamese,Malay,English": 0.5066666666666667, "Vietnamese,Indonesian,English": 0.5733333333333334, "Chinese,Spanish,Malay": 0.5466666666666666, "Chinese,Spanish,Indonesian": 0.6333333333333333, "Chinese,Spanish,English": 0.6533333333333333, "Chinese,Malay,Indonesian": 0.5533333333333333, "Chinese,Malay,English": 0.54, "Chinese,Indonesian,English": 0.6266666666666667, "Spanish,Malay,Indonesian": 0.5666666666666667, "Spanish,Malay,English": 0.5333333333333333, "Spanish,Indonesian,English": 0.6266666666666667, "Malay,Indonesian,English": 0.5333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.5133333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.4533333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.5066666666666667, "Filipino,Vietnamese,Chinese,English": 0.5266666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.43333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.5066666666666667, "Filipino,Vietnamese,Spanish,English": 0.5133333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.44666666666666666, "Filipino,Vietnamese,Malay,English": 0.43333333333333335, "Filipino,Vietnamese,Indonesian,English": 0.5, "Filipino,Chinese,Spanish,Malay": 0.4666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.5266666666666666, "Filipino,Chinese,Spanish,English": 0.54, "Filipino,Chinese,Malay,Indonesian": 0.4666666666666667, "Filipino,Chinese,Malay,English": 0.4533333333333333, "Filipino,Chinese,Indonesian,English": 0.52, "Filipino,Spanish,Malay,Indonesian": 0.47333333333333333, "Filipino,Spanish,Malay,English": 0.44666666666666666, "Filipino,Spanish,Indonesian,English": 0.5266666666666666, "Filipino,Malay,Indonesian,English": 0.44666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.48, "Vietnamese,Chinese,Spanish,Indonesian": 0.5533333333333333, "Vietnamese,Chinese,Spanish,English": 0.5666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Chinese,Malay,English": 0.48, "Vietnamese,Chinese,Indonesian,English": 0.5333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Spanish,Malay,English": 0.47333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.5533333333333333, "Vietnamese,Malay,Indonesian,English": 0.48, "Chinese,Spanish,Malay,Indonesian": 0.52, "Chinese,Spanish,Malay,English": 0.49333333333333335, "Chinese,Spanish,Indonesian,English": 0.5733333333333334, "Chinese,Malay,Indonesian,English": 0.5066666666666667, "Spanish,Malay,Indonesian,English": 0.5 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4866666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.49333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Malay,English": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.48, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.4866666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.42, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.4266666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.49333333333333335, "Filipino,Chinese,Malay,Indonesian,English": 0.43333333333333335, "Filipino,Spanish,Malay,Indonesian,English": 0.4266666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.47333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.52, "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian,English": 0.46, "Chinese,Spanish,Malay,Indonesian,English": 0.48 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.42, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.41333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.4066666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.41333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4 } }, "AC3_2": 0.6680188710765035, "AC3_3": 0.605520328962794, "AC3_4": 0.5672141524761157, "AC3_5": 0.539361197942404, "AC3_6": 0.5171662489080889, "AC3_7": 0.49838420103022407 }, "prompt_3": { "overall_acc": 0.6257142857142857, "language_acc": { "Filipino": 0.62, "Vietnamese": 0.6133333333333333, "Chinese": 0.6, "Spanish": 0.6666666666666666, "Malay": 0.56, "Indonesian": 0.6133333333333333, "English": 0.7066666666666667 }, "consistency_score_2": 0.5933333333333333, "consistency_score_3": 0.45980952380952383, "consistency_score_4": 0.3975238095238096, "consistency_score_5": 0.36000000000000004, "consistency_score_6": 0.3333333333333333, "consistency_score_7": 0.31333333333333335, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5866666666666667, "Filipino,Chinese": 0.5866666666666667, "Filipino,Spanish": 0.5866666666666667, "Filipino,Malay": 0.54, "Filipino,Indonesian": 0.6066666666666667, "Filipino,English": 0.6, "Vietnamese,Chinese": 0.6333333333333333, "Vietnamese,Spanish": 0.6333333333333333, "Vietnamese,Malay": 0.5466666666666666, "Vietnamese,Indonesian": 0.5666666666666667, "Vietnamese,English": 0.6066666666666667, "Chinese,Spanish": 0.62, "Chinese,Malay": 0.5666666666666667, "Chinese,Indonesian": 0.5933333333333334, "Chinese,English": 0.6, "Spanish,Malay": 0.5533333333333333, "Spanish,Indonesian": 0.6266666666666667, "Spanish,English": 0.6666666666666666, "Malay,Indonesian": 0.5933333333333334, "Malay,English": 0.5666666666666667, "Indonesian,English": 0.58 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.46, "Filipino,Vietnamese,Spanish": 0.47333333333333333, "Filipino,Vietnamese,Malay": 0.4066666666666667, "Filipino,Vietnamese,Indonesian": 0.46, "Filipino,Vietnamese,English": 0.47333333333333333, "Filipino,Chinese,Spanish": 0.47333333333333333, "Filipino,Chinese,Malay": 0.42, "Filipino,Chinese,Indonesian": 0.4666666666666667, "Filipino,Chinese,English": 0.46, "Filipino,Spanish,Malay": 0.4266666666666667, "Filipino,Spanish,Indonesian": 0.4866666666666667, "Filipino,Spanish,English": 0.49333333333333335, "Filipino,Malay,Indonesian": 0.44, "Filipino,Malay,English": 0.42, "Filipino,Indonesian,English": 0.4533333333333333, "Vietnamese,Chinese,Spanish": 0.49333333333333335, "Vietnamese,Chinese,Malay": 0.43333333333333335, "Vietnamese,Chinese,Indonesian": 0.4666666666666667, "Vietnamese,Chinese,English": 0.5066666666666667, "Vietnamese,Spanish,Malay": 0.43333333333333335, "Vietnamese,Spanish,Indonesian": 0.46, "Vietnamese,Spanish,English": 0.5066666666666667, "Vietnamese,Malay,Indonesian": 0.43333333333333335, "Vietnamese,Malay,English": 0.43333333333333335, "Vietnamese,Indonesian,English": 0.4533333333333333, "Chinese,Spanish,Malay": 0.4533333333333333, "Chinese,Spanish,Indonesian": 0.4866666666666667, "Chinese,Spanish,English": 0.5133333333333333, "Chinese,Malay,Indonesian": 0.44, "Chinese,Malay,English": 0.4533333333333333, "Chinese,Indonesian,English": 0.46, "Spanish,Malay,Indonesian": 0.4533333333333333, "Spanish,Malay,English": 0.47333333333333333, "Spanish,Indonesian,English": 0.4866666666666667, "Malay,Indonesian,English": 0.44 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.36666666666666664, "Filipino,Vietnamese,Chinese,Indonesian": 0.4, "Filipino,Vietnamese,Chinese,English": 0.4066666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Spanish,English": 0.42, "Filipino,Vietnamese,Malay,Indonesian": 0.38, "Filipino,Vietnamese,Malay,English": 0.36666666666666664, "Filipino,Vietnamese,Indonesian,English": 0.4, "Filipino,Chinese,Spanish,Malay": 0.38, "Filipino,Chinese,Spanish,Indonesian": 0.43333333333333335, "Filipino,Chinese,Spanish,English": 0.4266666666666667, "Filipino,Chinese,Malay,Indonesian": 0.37333333333333335, "Filipino,Chinese,Malay,English": 0.36666666666666664, "Filipino,Chinese,Indonesian,English": 0.4, "Filipino,Spanish,Malay,Indonesian": 0.38666666666666666, "Filipino,Spanish,Malay,English": 0.38666666666666666, "Filipino,Spanish,Indonesian,English": 0.42, "Filipino,Malay,Indonesian,English": 0.37333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.4, "Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, "Vietnamese,Chinese,Malay,English": 0.4066666666666667, "Vietnamese,Chinese,Indonesian,English": 0.41333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Spanish,Malay,English": 0.4, "Vietnamese,Spanish,Indonesian,English": 0.41333333333333333, "Vietnamese,Malay,Indonesian,English": 0.38, "Chinese,Spanish,Malay,Indonesian": 0.38666666666666666, "Chinese,Spanish,Malay,English": 0.42, "Chinese,Spanish,Indonesian,English": 0.42, "Chinese,Malay,Indonesian,English": 0.38, "Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.3466666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.38, "Filipino,Vietnamese,Chinese,Spanish,English": 0.38, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.3466666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.36666666666666664, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.3466666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.3466666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.37333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.3466666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.35333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.35333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.38666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.34, "Filipino,Spanish,Malay,Indonesian,English": 0.35333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.38666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.36, "Vietnamese,Spanish,Malay,Indonesian,English": 0.35333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.35333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.3333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.35333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.32666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.32666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.34 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 } }, "AC3_2": 0.6090937499500352, "AC3_3": 0.5300839745833088, "AC3_4": 0.48617487496578315, "AC3_5": 0.45704347821450286, "AC3_6": 0.43495531276497484, "AC3_7": 0.4175659228764255 }, "prompt_4": { "overall_acc": 0.66, "language_acc": { "Filipino": 0.6133333333333333, "Vietnamese": 0.6266666666666667, "Chinese": 0.6666666666666666, "Spanish": 0.7, "Malay": 0.6266666666666667, "Indonesian": 0.6333333333333333, "English": 0.7533333333333333 }, "consistency_score_2": 0.6676190476190476, "consistency_score_3": 0.5506666666666666, "consistency_score_4": 0.4876190476190476, "consistency_score_5": 0.44507936507936513, "consistency_score_6": 0.4123809523809524, "consistency_score_7": 0.38666666666666666, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6733333333333333, "Filipino,Chinese": 0.66, "Filipino,Spanish": 0.58, "Filipino,Malay": 0.62, "Filipino,Indonesian": 0.6133333333333333, "Filipino,English": 0.6333333333333333, "Vietnamese,Chinese": 0.66, "Vietnamese,Spanish": 0.6466666666666666, "Vietnamese,Malay": 0.64, "Vietnamese,Indonesian": 0.6333333333333333, "Vietnamese,English": 0.6733333333333333, "Chinese,Spanish": 0.7066666666666667, "Chinese,Malay": 0.6333333333333333, "Chinese,Indonesian": 0.6666666666666666, "Chinese,English": 0.7133333333333334, "Spanish,Malay": 0.7, "Spanish,Indonesian": 0.72, "Spanish,English": 0.7866666666666666, "Malay,Indonesian": 0.6933333333333334, "Malay,English": 0.6733333333333333, "Indonesian,English": 0.6933333333333334 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5466666666666666, "Filipino,Vietnamese,Spanish": 0.49333333333333335, "Filipino,Vietnamese,Malay": 0.5066666666666667, "Filipino,Vietnamese,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,English": 0.54, "Filipino,Chinese,Spanish": 0.5266666666666666, "Filipino,Chinese,Malay": 0.5133333333333333, "Filipino,Chinese,Indonesian": 0.5266666666666666, "Filipino,Chinese,English": 0.5533333333333333, "Filipino,Spanish,Malay": 0.49333333333333335, "Filipino,Spanish,Indonesian": 0.5, "Filipino,Spanish,English": 0.5466666666666666, "Filipino,Malay,Indonesian": 0.52, "Filipino,Malay,English": 0.52, "Filipino,Indonesian,English": 0.5333333333333333, "Vietnamese,Chinese,Spanish": 0.56, "Vietnamese,Chinese,Malay": 0.5333333333333333, "Vietnamese,Chinese,Indonesian": 0.54, "Vietnamese,Chinese,English": 0.58, "Vietnamese,Spanish,Malay": 0.54, "Vietnamese,Spanish,Indonesian": 0.5333333333333333, "Vietnamese,Spanish,English": 0.5933333333333334, "Vietnamese,Malay,Indonesian": 0.5266666666666666, "Vietnamese,Malay,English": 0.5266666666666666, "Vietnamese,Indonesian,English": 0.5533333333333333, "Chinese,Spanish,Malay": 0.5733333333333334, "Chinese,Spanish,Indonesian": 0.5866666666666667, "Chinese,Spanish,English": 0.6533333333333333, "Chinese,Malay,Indonesian": 0.5533333333333333, "Chinese,Malay,English": 0.58, "Chinese,Indonesian,English": 0.6, "Spanish,Malay,Indonesian": 0.5933333333333334, "Spanish,Malay,English": 0.6066666666666667, "Spanish,Indonesian,English": 0.6266666666666667, "Malay,Indonesian,English": 0.58 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.47333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.4666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.4666666666666667, "Filipino,Vietnamese,Chinese,English": 0.49333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.43333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.44666666666666666, "Filipino,Vietnamese,Spanish,English": 0.47333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.44666666666666666, "Filipino,Vietnamese,Malay,English": 0.44666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.47333333333333333, "Filipino,Chinese,Spanish,Malay": 0.4666666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.47333333333333333, "Filipino,Chinese,Spanish,English": 0.5066666666666667, "Filipino,Chinese,Malay,Indonesian": 0.4666666666666667, "Filipino,Chinese,Malay,English": 0.48, "Filipino,Chinese,Indonesian,English": 0.49333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.4533333333333333, "Filipino,Spanish,Malay,English": 0.4666666666666667, "Filipino,Spanish,Indonesian,English": 0.4866666666666667, "Filipino,Malay,Indonesian,English": 0.47333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.4866666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.49333333333333335, "Vietnamese,Chinese,Spanish,English": 0.54, "Vietnamese,Chinese,Malay,Indonesian": 0.4866666666666667, "Vietnamese,Chinese,Malay,English": 0.49333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.5133333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.48, "Vietnamese,Spanish,Malay,English": 0.49333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.5133333333333333, "Vietnamese,Malay,Indonesian,English": 0.4866666666666667, "Chinese,Spanish,Malay,Indonesian": 0.52, "Chinese,Spanish,Malay,English": 0.5466666666666666, "Chinese,Spanish,Indonesian,English": 0.5533333333333333, "Chinese,Malay,Indonesian,English": 0.5266666666666666, "Spanish,Malay,Indonesian,English": 0.5466666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.4533333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.44666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.43333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.42, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.44, "Filipino,Chinese,Spanish,Malay,English": 0.44666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.46, "Filipino,Chinese,Malay,Indonesian,English": 0.44666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.44, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4533333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.4666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.48, "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, "Vietnamese,Spanish,Malay,Indonesian,English": 0.46, "Chinese,Spanish,Malay,Indonesian,English": 0.5066666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.4066666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3933333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.4266666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.44 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 } }, "AC3_2": 0.6637876613560274, "AC3_3": 0.6003964757213329, "AC3_4": 0.5608630704905473, "AC3_5": 0.5316403331894667, "AC3_6": 0.5076021313913871, "AC3_7": 0.4876433120553207 }, "prompt_5": { "overall_acc": 0.6476190476190476, "language_acc": { "Filipino": 0.6666666666666666, "Vietnamese": 0.6133333333333333, "Chinese": 0.6266666666666667, "Spanish": 0.6933333333333334, "Malay": 0.6, "Indonesian": 0.64, "English": 0.6933333333333334 }, "consistency_score_2": 0.6301587301587301, "consistency_score_3": 0.5059047619047619, "consistency_score_4": 0.44704761904761897, "consistency_score_5": 0.4111111111111111, "consistency_score_6": 0.38571428571428573, "consistency_score_7": 0.36666666666666664, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.6133333333333333, "Filipino,Chinese": 0.58, "Filipino,Spanish": 0.6733333333333333, "Filipino,Malay": 0.6133333333333333, "Filipino,Indonesian": 0.6133333333333333, "Filipino,English": 0.64, "Vietnamese,Chinese": 0.6133333333333333, "Vietnamese,Spanish": 0.6466666666666666, "Vietnamese,Malay": 0.6, "Vietnamese,Indonesian": 0.6666666666666666, "Vietnamese,English": 0.6333333333333333, "Chinese,Spanish": 0.6533333333333333, "Chinese,Malay": 0.5133333333333333, "Chinese,Indonesian": 0.58, "Chinese,English": 0.68, "Spanish,Malay": 0.5933333333333334, "Spanish,Indonesian": 0.6933333333333334, "Spanish,English": 0.7533333333333333, "Malay,Indonesian": 0.62, "Malay,English": 0.5933333333333334, "Indonesian,English": 0.66 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.4666666666666667, "Filipino,Vietnamese,Spanish": 0.5333333333333333, "Filipino,Vietnamese,Malay": 0.47333333333333333, "Filipino,Vietnamese,Indonesian": 0.5133333333333333, "Filipino,Vietnamese,English": 0.5, "Filipino,Chinese,Spanish": 0.5066666666666667, "Filipino,Chinese,Malay": 0.42, "Filipino,Chinese,Indonesian": 0.46, "Filipino,Chinese,English": 0.5133333333333333, "Filipino,Spanish,Malay": 0.5, "Filipino,Spanish,Indonesian": 0.5533333333333333, "Filipino,Spanish,English": 0.5733333333333334, "Filipino,Malay,Indonesian": 0.4866666666666667, "Filipino,Malay,English": 0.49333333333333335, "Filipino,Indonesian,English": 0.5266666666666666, "Vietnamese,Chinese,Spanish": 0.5066666666666667, "Vietnamese,Chinese,Malay": 0.44, "Vietnamese,Chinese,Indonesian": 0.4866666666666667, "Vietnamese,Chinese,English": 0.52, "Vietnamese,Spanish,Malay": 0.48, "Vietnamese,Spanish,Indonesian": 0.5533333333333333, "Vietnamese,Spanish,English": 0.56, "Vietnamese,Malay,Indonesian": 0.49333333333333335, "Vietnamese,Malay,English": 0.46, "Vietnamese,Indonesian,English": 0.54, "Chinese,Spanish,Malay": 0.44666666666666666, "Chinese,Spanish,Indonesian": 0.52, "Chinese,Spanish,English": 0.5866666666666667, "Chinese,Malay,Indonesian": 0.4533333333333333, "Chinese,Malay,English": 0.4533333333333333, "Chinese,Indonesian,English": 0.5266666666666666, "Spanish,Malay,Indonesian": 0.5266666666666666, "Spanish,Malay,English": 0.5133333333333333, "Spanish,Indonesian,English": 0.6, "Malay,Indonesian,English": 0.52 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.44, "Filipino,Vietnamese,Chinese,Malay": 0.3933333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.42, "Filipino,Vietnamese,Chinese,English": 0.44, "Filipino,Vietnamese,Spanish,Malay": 0.43333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.4866666666666667, "Filipino,Vietnamese,Spanish,English": 0.48, "Filipino,Vietnamese,Malay,Indonesian": 0.4266666666666667, "Filipino,Vietnamese,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.46, "Filipino,Chinese,Spanish,Malay": 0.4066666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.4533333333333333, "Filipino,Chinese,Spanish,English": 0.4866666666666667, "Filipino,Chinese,Malay,Indonesian": 0.4, "Filipino,Chinese,Malay,English": 0.4066666666666667, "Filipino,Chinese,Indonesian,English": 0.4533333333333333, "Filipino,Spanish,Malay,Indonesian": 0.4533333333333333, "Filipino,Spanish,Malay,English": 0.4533333333333333, "Filipino,Spanish,Indonesian,English": 0.5133333333333333, "Filipino,Malay,Indonesian,English": 0.44666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.4066666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.4533333333333333, "Vietnamese,Chinese,Spanish,English": 0.4866666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.42, "Vietnamese,Chinese,Malay,English": 0.41333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.4666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.4533333333333333, "Vietnamese,Spanish,Malay,English": 0.44, "Vietnamese,Spanish,Indonesian,English": 0.5066666666666667, "Vietnamese,Malay,Indonesian,English": 0.44666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.43333333333333335, "Chinese,Spanish,Malay,English": 0.43333333333333335, "Chinese,Spanish,Indonesian,English": 0.5, "Chinese,Malay,Indonesian,English": 0.43333333333333335, "Spanish,Malay,Indonesian,English": 0.4866666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.38, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.4266666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.38, "Filipino,Vietnamese,Chinese,Malay,English": 0.38, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.41333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.41333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.4, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.4533333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.4, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.3933333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.4, "Filipino,Chinese,Spanish,Indonesian,English": 0.44666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.3933333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.43333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4, "Vietnamese,Chinese,Spanish,Malay,English": 0.4, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.44666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.4066666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.43333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.42 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.37333333333333335, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3933333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36666666666666664 } }, "AC3_2": 0.6387695947444489, "AC3_3": 0.5680568660261147, "AC3_4": 0.5289583798931801, "AC3_5": 0.5029485256896266, "AC3_6": 0.48347597098678674, "AC3_7": 0.46823161184742007 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.6249999999999999, "language_acc": { "Indonesian": 0.5397727272727273, "English": 0.75, "Filipino": 0.48863636363636365, "Spanish": 0.6477272727272727, "Chinese": 0.75, "Malay": 0.5852272727272727, "Vietnamese": 0.6136363636363636 }, "consistency_score_2": 0.689935064935065, "consistency_score_3": 0.5577922077922078, "consistency_score_4": 0.47727272727272724, "consistency_score_5": 0.42099567099567103, "consistency_score_6": 0.37905844155844154, "consistency_score_7": 0.3465909090909091, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6931818181818182, "Indonesian,Filipino": 0.6477272727272727, "Indonesian,Spanish": 0.7443181818181818, "Indonesian,Chinese": 0.6818181818181818, "Indonesian,Malay": 0.7840909090909091, "Indonesian,Vietnamese": 0.7329545454545454, "English,Filipino": 0.6022727272727273, "English,Spanish": 0.7897727272727273, "English,Chinese": 0.7386363636363636, "English,Malay": 0.7102272727272727, "English,Vietnamese": 0.7840909090909091, "Filipino,Spanish": 0.6477272727272727, "Filipino,Chinese": 0.5454545454545454, "Filipino,Malay": 0.625, "Filipino,Vietnamese": 0.6136363636363636, "Spanish,Chinese": 0.6704545454545454, "Spanish,Malay": 0.7215909090909091, "Spanish,Vietnamese": 0.7727272727272727, "Chinese,Malay": 0.6420454545454546, "Chinese,Vietnamese": 0.6534090909090909, "Malay,Vietnamese": 0.6875 }, "3_combine": { "Indonesian,English,Filipino": 0.5056818181818182, "Indonesian,English,Spanish": 0.625, "Indonesian,English,Chinese": 0.5738636363636364, "Indonesian,English,Malay": 0.6079545454545454, "Indonesian,English,Vietnamese": 0.6193181818181818, "Indonesian,Filipino,Spanish": 0.5568181818181818, "Indonesian,Filipino,Chinese": 0.4715909090909091, "Indonesian,Filipino,Malay": 0.5454545454545454, "Indonesian,Filipino,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Chinese": 0.5795454545454546, "Indonesian,Spanish,Malay": 0.6420454545454546, "Indonesian,Spanish,Vietnamese": 0.6363636363636364, "Indonesian,Chinese,Malay": 0.5738636363636364, "Indonesian,Chinese,Vietnamese": 0.5511363636363636, "Indonesian,Malay,Vietnamese": 0.6136363636363636, "English,Filipino,Spanish": 0.5454545454545454, "English,Filipino,Chinese": 0.4772727272727273, "English,Filipino,Malay": 0.4943181818181818, "English,Filipino,Vietnamese": 0.5284090909090909, "English,Spanish,Chinese": 0.6136363636363636, "English,Spanish,Malay": 0.625, "English,Spanish,Vietnamese": 0.6818181818181818, "English,Chinese,Malay": 0.5625, "English,Chinese,Vietnamese": 0.6022727272727273, "English,Malay,Vietnamese": 0.6079545454545454, "Filipino,Spanish,Chinese": 0.4659090909090909, "Filipino,Spanish,Malay": 0.5284090909090909, "Filipino,Spanish,Vietnamese": 0.5454545454545454, "Filipino,Chinese,Malay": 0.44886363636363635, "Filipino,Chinese,Vietnamese": 0.4318181818181818, "Filipino,Malay,Vietnamese": 0.5, "Spanish,Chinese,Malay": 0.5397727272727273, "Spanish,Chinese,Vietnamese": 0.5738636363636364, "Spanish,Malay,Vietnamese": 0.6079545454545454, "Chinese,Malay,Vietnamese": 0.5170454545454546 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.4715909090909091, "Indonesian,English,Filipino,Chinese": 0.42613636363636365, "Indonesian,English,Filipino,Malay": 0.44886363636363635, "Indonesian,English,Filipino,Vietnamese": 0.4715909090909091, "Indonesian,English,Spanish,Chinese": 0.5284090909090909, "Indonesian,English,Spanish,Malay": 0.5625, "Indonesian,English,Spanish,Vietnamese": 0.5681818181818182, "Indonesian,English,Chinese,Malay": 0.5170454545454546, "Indonesian,English,Chinese,Vietnamese": 0.5113636363636364, "Indonesian,English,Malay,Vietnamese": 0.5454545454545454, "Indonesian,Filipino,Spanish,Chinese": 0.4318181818181818, "Indonesian,Filipino,Spanish,Malay": 0.4943181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.4943181818181818, "Indonesian,Filipino,Chinese,Malay": 0.4147727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.4602272727272727, "Indonesian,Spanish,Chinese,Malay": 0.5056818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.5056818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.5625, "Indonesian,Chinese,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Spanish,Chinese": 0.4375, "English,Filipino,Spanish,Malay": 0.4659090909090909, "English,Filipino,Spanish,Vietnamese": 0.4943181818181818, "English,Filipino,Chinese,Malay": 0.4090909090909091, "English,Filipino,Chinese,Vietnamese": 0.42045454545454547, "English,Filipino,Malay,Vietnamese": 0.45454545454545453, "English,Spanish,Chinese,Malay": 0.5056818181818182, "English,Spanish,Chinese,Vietnamese": 0.5397727272727273, "English,Spanish,Malay,Vietnamese": 0.5511363636363636, "English,Chinese,Malay,Vietnamese": 0.48863636363636365, "Filipino,Spanish,Chinese,Malay": 0.4034090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.4090909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.4659090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.375, "Spanish,Chinese,Malay,Vietnamese": 0.4715909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4034090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.4318181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.44886363636363635, "Indonesian,English,Filipino,Chinese,Malay": 0.38636363636363635, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.3977272727272727, "Indonesian,English,Filipino,Malay,Vietnamese": 0.42045454545454547, "Indonesian,English,Spanish,Chinese,Malay": 0.4772727272727273, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4772727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5113636363636364, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4602272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.38636363636363635, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.39204545454545453, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.36363636363636365, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.44886363636363635, "English,Filipino,Spanish,Chinese,Malay": 0.38636363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.3977272727272727, "English,Filipino,Spanish,Malay,Vietnamese": 0.4318181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.3693181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.44886363636363635, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.35795454545454547 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3693181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.3806818181818182, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4090909090909091, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.35795454545454547, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4318181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3522727272727273 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3465909090909091 } }, "AC3_2": 0.655864197480986, "AC3_3": 0.5894866867467576, "AC3_4": 0.5412371133529599, "AC3_5": 0.5031039833973234, "AC3_6": 0.4719078415051422, "AC3_7": 0.4459064327026435 }, "prompt_2": { "overall_acc": 0.6176948051948052, "language_acc": { "Indonesian": 0.5681818181818182, "English": 0.7329545454545454, "Filipino": 0.4943181818181818, "Spanish": 0.6306818181818182, "Chinese": 0.7329545454545454, "Malay": 0.5681818181818182, "Vietnamese": 0.5965909090909091 }, "consistency_score_2": 0.6988636363636364, "consistency_score_3": 0.5712662337662338, "consistency_score_4": 0.49350649350649345, "consistency_score_5": 0.43858225108225113, "consistency_score_6": 0.3969155844155844, "consistency_score_7": 0.36363636363636365, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7215909090909091, "Indonesian,Filipino": 0.6704545454545454, "Indonesian,Spanish": 0.7670454545454546, "Indonesian,Chinese": 0.6988636363636364, "Indonesian,Malay": 0.8011363636363636, "Indonesian,Vietnamese": 0.75, "English,Filipino": 0.6079545454545454, "English,Spanish": 0.7840909090909091, "English,Chinese": 0.7215909090909091, "English,Malay": 0.6761363636363636, "English,Vietnamese": 0.75, "Filipino,Spanish": 0.6590909090909091, "Filipino,Chinese": 0.5852272727272727, "Filipino,Malay": 0.6306818181818182, "Filipino,Vietnamese": 0.625, "Spanish,Chinese": 0.6704545454545454, "Spanish,Malay": 0.7386363636363636, "Spanish,Vietnamese": 0.7897727272727273, "Chinese,Malay": 0.6534090909090909, "Chinese,Vietnamese": 0.6420454545454546, "Malay,Vietnamese": 0.7329545454545454 }, "3_combine": { "Indonesian,English,Filipino": 0.5397727272727273, "Indonesian,English,Spanish": 0.6477272727272727, "Indonesian,English,Chinese": 0.6079545454545454, "Indonesian,English,Malay": 0.6079545454545454, "Indonesian,English,Vietnamese": 0.6193181818181818, "Indonesian,Filipino,Spanish": 0.5795454545454546, "Indonesian,Filipino,Chinese": 0.5056818181818182, "Indonesian,Filipino,Malay": 0.5625, "Indonesian,Filipino,Vietnamese": 0.5454545454545454, "Indonesian,Spanish,Chinese": 0.5965909090909091, "Indonesian,Spanish,Malay": 0.6590909090909091, "Indonesian,Spanish,Vietnamese": 0.6704545454545454, "Indonesian,Chinese,Malay": 0.6022727272727273, "Indonesian,Chinese,Vietnamese": 0.5625, "Indonesian,Malay,Vietnamese": 0.6477272727272727, "English,Filipino,Spanish": 0.5397727272727273, "English,Filipino,Chinese": 0.48863636363636365, "English,Filipino,Malay": 0.48295454545454547, "English,Filipino,Vietnamese": 0.5170454545454546, "English,Spanish,Chinese": 0.6193181818181818, "English,Spanish,Malay": 0.6136363636363636, "English,Spanish,Vietnamese": 0.6704545454545454, "English,Chinese,Malay": 0.5681818181818182, "English,Chinese,Vietnamese": 0.5909090909090909, "English,Malay,Vietnamese": 0.5909090909090909, "Filipino,Spanish,Chinese": 0.48295454545454547, "Filipino,Spanish,Malay": 0.5397727272727273, "Filipino,Spanish,Vietnamese": 0.5568181818181818, "Filipino,Chinese,Malay": 0.4772727272727273, "Filipino,Chinese,Vietnamese": 0.44886363636363635, "Filipino,Malay,Vietnamese": 0.5170454545454546, "Spanish,Chinese,Malay": 0.5625, "Spanish,Chinese,Vietnamese": 0.5795454545454546, "Spanish,Malay,Vietnamese": 0.6477272727272727, "Chinese,Malay,Vietnamese": 0.5454545454545454 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.48863636363636365, "Indonesian,English,Filipino,Chinese": 0.45454545454545453, "Indonesian,English,Filipino,Malay": 0.4602272727272727, "Indonesian,English,Filipino,Vietnamese": 0.4715909090909091, "Indonesian,English,Spanish,Chinese": 0.5568181818181818, "Indonesian,English,Spanish,Malay": 0.5738636363636364, "Indonesian,English,Spanish,Vietnamese": 0.5795454545454546, "Indonesian,English,Chinese,Malay": 0.5397727272727273, "Indonesian,English,Chinese,Vietnamese": 0.5284090909090909, "Indonesian,English,Malay,Vietnamese": 0.5454545454545454, "Indonesian,Filipino,Spanish,Chinese": 0.4602272727272727, "Indonesian,Filipino,Spanish,Malay": 0.5170454545454546, "Indonesian,Filipino,Spanish,Vietnamese": 0.5113636363636364, "Indonesian,Filipino,Chinese,Malay": 0.44886363636363635, "Indonesian,Filipino,Chinese,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Spanish,Chinese,Malay": 0.5397727272727273, "Indonesian,Spanish,Chinese,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.5965909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.5113636363636364, "English,Filipino,Spanish,Chinese": 0.44886363636363635, "English,Filipino,Spanish,Malay": 0.4659090909090909, "English,Filipino,Spanish,Vietnamese": 0.4772727272727273, "English,Filipino,Chinese,Malay": 0.42045454545454547, "English,Filipino,Chinese,Vietnamese": 0.42613636363636365, "English,Filipino,Malay,Vietnamese": 0.4431818181818182, "English,Spanish,Chinese,Malay": 0.5227272727272727, "English,Spanish,Chinese,Vietnamese": 0.5511363636363636, "English,Spanish,Malay,Vietnamese": 0.5625, "English,Chinese,Malay,Vietnamese": 0.5, "Filipino,Spanish,Chinese,Malay": 0.4318181818181818, "Filipino,Spanish,Chinese,Vietnamese": 0.42045454545454547, "Filipino,Spanish,Malay,Vietnamese": 0.48863636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.4034090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.5056818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.42613636363636365, "Indonesian,English,Filipino,Spanish,Malay": 0.44886363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.4431818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.4090909090909091, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.42045454545454547, "Indonesian,English,Spanish,Chinese,Malay": 0.5056818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.5, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5284090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4772727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.42613636363636365, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4659090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.38636363636363635, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Spanish,Chinese,Malay": 0.4034090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.4318181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.3806818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.4772727272727273, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3977272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4147727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3693181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4602272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3806818181818182, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.36363636363636365 } }, "AC3_2": 0.655777098930125, "AC3_3": 0.5935740085466136, "AC3_4": 0.5486609810440379, "AC3_5": 0.5129524995522468, "AC3_6": 0.4832844155367831, "AC3_7": 0.45777878031277675 }, "prompt_3": { "overall_acc": 0.6217532467532468, "language_acc": { "Indonesian": 0.5795454545454546, "English": 0.7443181818181818, "Filipino": 0.5056818181818182, "Spanish": 0.6420454545454546, "Chinese": 0.7045454545454546, "Malay": 0.5568181818181818, "Vietnamese": 0.6193181818181818 }, "consistency_score_2": 0.6888528138528138, "consistency_score_3": 0.5603896103896104, "consistency_score_4": 0.48214285714285715, "consistency_score_5": 0.4266774891774892, "consistency_score_6": 0.3831168831168831, "consistency_score_7": 0.3465909090909091, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.75, "Indonesian,Filipino": 0.6420454545454546, "Indonesian,Spanish": 0.7613636363636364, "Indonesian,Chinese": 0.6875, "Indonesian,Malay": 0.7954545454545454, "Indonesian,Vietnamese": 0.7272727272727273, "English,Filipino": 0.5795454545454546, "English,Spanish": 0.7897727272727273, "English,Chinese": 0.7386363636363636, "English,Malay": 0.7102272727272727, "English,Vietnamese": 0.7215909090909091, "Filipino,Spanish": 0.6420454545454546, "Filipino,Chinese": 0.5568181818181818, "Filipino,Malay": 0.6136363636363636, "Filipino,Vietnamese": 0.5909090909090909, "Spanish,Chinese": 0.6875, "Spanish,Malay": 0.7386363636363636, "Spanish,Vietnamese": 0.7329545454545454, "Chinese,Malay": 0.6647727272727273, "Chinese,Vietnamese": 0.6306818181818182, "Malay,Vietnamese": 0.7045454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.5056818181818182, "Indonesian,English,Spanish": 0.6647727272727273, "Indonesian,English,Chinese": 0.625, "Indonesian,English,Malay": 0.6420454545454546, "Indonesian,English,Vietnamese": 0.6136363636363636, "Indonesian,Filipino,Spanish": 0.5568181818181818, "Indonesian,Filipino,Chinese": 0.4715909090909091, "Indonesian,Filipino,Malay": 0.5454545454545454, "Indonesian,Filipino,Vietnamese": 0.5, "Indonesian,Spanish,Chinese": 0.6136363636363636, "Indonesian,Spanish,Malay": 0.6647727272727273, "Indonesian,Spanish,Vietnamese": 0.6363636363636364, "Indonesian,Chinese,Malay": 0.6022727272727273, "Indonesian,Chinese,Vietnamese": 0.5568181818181818, "Indonesian,Malay,Vietnamese": 0.6306818181818182, "English,Filipino,Spanish": 0.5284090909090909, "English,Filipino,Chinese": 0.4772727272727273, "English,Filipino,Malay": 0.48295454545454547, "English,Filipino,Vietnamese": 0.4659090909090909, "English,Spanish,Chinese": 0.6420454545454546, "English,Spanish,Malay": 0.6363636363636364, "English,Spanish,Vietnamese": 0.6363636363636364, "English,Chinese,Malay": 0.5795454545454546, "English,Chinese,Vietnamese": 0.5795454545454546, "English,Malay,Vietnamese": 0.5909090909090909, "Filipino,Spanish,Chinese": 0.48295454545454547, "Filipino,Spanish,Malay": 0.5284090909090909, "Filipino,Spanish,Vietnamese": 0.5056818181818182, "Filipino,Chinese,Malay": 0.45454545454545453, "Filipino,Chinese,Vietnamese": 0.42045454545454547, "Filipino,Malay,Vietnamese": 0.48863636363636365, "Spanish,Chinese,Malay": 0.5738636363636364, "Spanish,Chinese,Vietnamese": 0.5681818181818182, "Spanish,Malay,Vietnamese": 0.6136363636363636, "Chinese,Malay,Vietnamese": 0.5284090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.48295454545454547, "Indonesian,English,Filipino,Chinese": 0.4375, "Indonesian,English,Filipino,Malay": 0.45454545454545453, "Indonesian,English,Filipino,Vietnamese": 0.42613636363636365, "Indonesian,English,Spanish,Chinese": 0.5852272727272727, "Indonesian,English,Spanish,Malay": 0.5965909090909091, "Indonesian,English,Spanish,Vietnamese": 0.5681818181818182, "Indonesian,English,Chinese,Malay": 0.5568181818181818, "Indonesian,English,Chinese,Vietnamese": 0.5284090909090909, "Indonesian,English,Malay,Vietnamese": 0.5511363636363636, "Indonesian,Filipino,Spanish,Chinese": 0.45454545454545453, "Indonesian,Filipino,Spanish,Malay": 0.4943181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.4602272727272727, "Indonesian,Filipino,Chinese,Malay": 0.4318181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.39204545454545453, "Indonesian,Filipino,Malay,Vietnamese": 0.4431818181818182, "Indonesian,Spanish,Chinese,Malay": 0.5511363636363636, "Indonesian,Spanish,Chinese,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.5738636363636364, "Indonesian,Chinese,Malay,Vietnamese": 0.5056818181818182, "English,Filipino,Spanish,Chinese": 0.45454545454545453, "English,Filipino,Spanish,Malay": 0.4659090909090909, "English,Filipino,Spanish,Vietnamese": 0.4375, "English,Filipino,Chinese,Malay": 0.4147727272727273, "English,Filipino,Chinese,Vietnamese": 0.3977272727272727, "English,Filipino,Malay,Vietnamese": 0.4147727272727273, "English,Spanish,Chinese,Malay": 0.5340909090909091, "English,Spanish,Chinese,Vietnamese": 0.5397727272727273, "English,Spanish,Malay,Vietnamese": 0.5511363636363636, "English,Chinese,Malay,Vietnamese": 0.4943181818181818, "Filipino,Spanish,Chinese,Malay": 0.42613636363636365, "Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.44886363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.375, "Spanish,Chinese,Malay,Vietnamese": 0.5 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4318181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.4431818181818182, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.4090909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.4034090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.3806818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.39204545454545453, "Indonesian,English,Spanish,Chinese,Malay": 0.5284090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.5056818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5227272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.48295454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4147727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.36363636363636365, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.48295454545454547, "English,Filipino,Spanish,Chinese,Malay": 0.4034090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.4034090909090909, "English,Filipino,Chinese,Malay,Vietnamese": 0.35795454545454547, "English,Spanish,Chinese,Malay,Vietnamese": 0.4715909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3977272727272727, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.375, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.3806818181818182, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3522727272727273, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4659090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.35795454545454547, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3522727272727273 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3465909090909091 } }, "AC3_2": 0.6535853699888455, "AC3_3": 0.5894787716590629, "AC3_4": 0.5431197478499595, "AC3_5": 0.5060670296961801, "AC3_6": 0.4740994062008529, "AC3_7": 0.4450773450732936 }, "prompt_4": { "overall_acc": 0.6217532467532468, "language_acc": { "Indonesian": 0.5738636363636364, "English": 0.7102272727272727, "Filipino": 0.5056818181818182, "Spanish": 0.6534090909090909, "Chinese": 0.7159090909090909, "Malay": 0.5795454545454546, "Vietnamese": 0.6136363636363636 }, "consistency_score_2": 0.6764069264069263, "consistency_score_3": 0.5441558441558441, "consistency_score_4": 0.46672077922077915, "consistency_score_5": 0.41369047619047616, "consistency_score_6": 0.3733766233766234, "consistency_score_7": 0.3409090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.6477272727272727, "Indonesian,Filipino": 0.6590909090909091, "Indonesian,Spanish": 0.75, "Indonesian,Chinese": 0.6818181818181818, "Indonesian,Malay": 0.7670454545454546, "Indonesian,Vietnamese": 0.7159090909090909, "English,Filipino": 0.5625, "English,Spanish": 0.7784090909090909, "English,Chinese": 0.7159090909090909, "English,Malay": 0.6818181818181818, "English,Vietnamese": 0.7215909090909091, "Filipino,Spanish": 0.6363636363636364, "Filipino,Chinese": 0.5511363636363636, "Filipino,Malay": 0.6193181818181818, "Filipino,Vietnamese": 0.6079545454545454, "Spanish,Chinese": 0.6761363636363636, "Spanish,Malay": 0.6988636363636364, "Spanish,Vietnamese": 0.7897727272727273, "Chinese,Malay": 0.6022727272727273, "Chinese,Vietnamese": 0.6306818181818182, "Malay,Vietnamese": 0.7102272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.4715909090909091, "Indonesian,English,Spanish": 0.6079545454545454, "Indonesian,English,Chinese": 0.5568181818181818, "Indonesian,English,Malay": 0.5795454545454546, "Indonesian,English,Vietnamese": 0.5738636363636364, "Indonesian,Filipino,Spanish": 0.5625, "Indonesian,Filipino,Chinese": 0.48863636363636365, "Indonesian,Filipino,Malay": 0.5397727272727273, "Indonesian,Filipino,Vietnamese": 0.5227272727272727, "Indonesian,Spanish,Chinese": 0.5852272727272727, "Indonesian,Spanish,Malay": 0.6363636363636364, "Indonesian,Spanish,Vietnamese": 0.6477272727272727, "Indonesian,Chinese,Malay": 0.5568181818181818, "Indonesian,Chinese,Vietnamese": 0.5397727272727273, "Indonesian,Malay,Vietnamese": 0.6136363636363636, "English,Filipino,Spanish": 0.5056818181818182, "English,Filipino,Chinese": 0.45454545454545453, "English,Filipino,Malay": 0.4659090909090909, "English,Filipino,Vietnamese": 0.4772727272727273, "English,Spanish,Chinese": 0.6079545454545454, "English,Spanish,Malay": 0.6022727272727273, "English,Spanish,Vietnamese": 0.6590909090909091, "English,Chinese,Malay": 0.5227272727272727, "English,Chinese,Vietnamese": 0.5625, "English,Malay,Vietnamese": 0.5795454545454546, "Filipino,Spanish,Chinese": 0.4715909090909091, "Filipino,Spanish,Malay": 0.5113636363636364, "Filipino,Spanish,Vietnamese": 0.5511363636363636, "Filipino,Chinese,Malay": 0.4431818181818182, "Filipino,Chinese,Vietnamese": 0.42613636363636365, "Filipino,Malay,Vietnamese": 0.5056818181818182, "Spanish,Chinese,Malay": 0.5284090909090909, "Spanish,Chinese,Vietnamese": 0.5738636363636364, "Spanish,Malay,Vietnamese": 0.6136363636363636, "Chinese,Malay,Vietnamese": 0.5 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.45454545454545453, "Indonesian,English,Filipino,Chinese": 0.4147727272727273, "Indonesian,English,Filipino,Malay": 0.4318181818181818, "Indonesian,English,Filipino,Vietnamese": 0.4318181818181818, "Indonesian,English,Spanish,Chinese": 0.5227272727272727, "Indonesian,English,Spanish,Malay": 0.5454545454545454, "Indonesian,English,Spanish,Vietnamese": 0.5511363636363636, "Indonesian,English,Chinese,Malay": 0.4943181818181818, "Indonesian,English,Chinese,Vietnamese": 0.48863636363636365, "Indonesian,English,Malay,Vietnamese": 0.5170454545454546, "Indonesian,Filipino,Spanish,Chinese": 0.45454545454545453, "Indonesian,Filipino,Spanish,Malay": 0.48295454545454547, "Indonesian,Filipino,Spanish,Vietnamese": 0.5, "Indonesian,Filipino,Chinese,Malay": 0.42613636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.45454545454545453, "Indonesian,Spanish,Chinese,Malay": 0.5113636363636364, "Indonesian,Spanish,Chinese,Vietnamese": 0.5056818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.5681818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.4772727272727273, "English,Filipino,Spanish,Chinese": 0.42045454545454547, "English,Filipino,Spanish,Malay": 0.4375, "English,Filipino,Spanish,Vietnamese": 0.4602272727272727, "English,Filipino,Chinese,Malay": 0.3806818181818182, "English,Filipino,Chinese,Vietnamese": 0.39204545454545453, "English,Filipino,Malay,Vietnamese": 0.4318181818181818, "English,Spanish,Chinese,Malay": 0.48295454545454547, "English,Spanish,Chinese,Vietnamese": 0.5284090909090909, "English,Spanish,Malay,Vietnamese": 0.5397727272727273, "English,Chinese,Malay,Vietnamese": 0.4602272727272727, "Filipino,Spanish,Chinese,Malay": 0.4090909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.42045454545454547, "Filipino,Spanish,Malay,Vietnamese": 0.4715909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.3806818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.4715909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.4034090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.4147727272727273, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.42613636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.375, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.3806818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.3977272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.4659090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4659090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.5, "Indonesian,English,Chinese,Malay,Vietnamese": 0.4375, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4034090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.375, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.45454545454545453, "English,Filipino,Spanish,Chinese,Malay": 0.3693181818181818, "English,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.4147727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.3522727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.4375, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.36363636363636365, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.375, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.39204545454545453, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3465909090909091, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3465909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3409090909090909 } }, "AC3_2": 0.6479296026525134, "AC3_3": 0.5803722870894258, "AC3_4": 0.5331962965701298, "AC3_5": 0.4968177236445792, "AC3_6": 0.466568504857672, "AC3_7": 0.4403648627474488 }, "prompt_5": -1 }, "sg_eval": { "prompt_1": { "accuracy": 0.6893203883495146 }, "prompt_2": { "accuracy": 0.6893203883495146 }, "prompt_3": { "accuracy": 0.6310679611650486 }, "prompt_4": { "accuracy": 0.7378640776699029 }, "prompt_5": { "accuracy": 0.6601941747572816 } }, "cn_eval": { "prompt_1": { "accuracy": 0.7619047619047619 }, "prompt_2": { "accuracy": 0.7904761904761904 }, "prompt_3": { "accuracy": 0.7428571428571429 }, "prompt_4": { "accuracy": 0.780952380952381 }, "prompt_5": { "accuracy": 0.819047619047619 } }, "us_eval": { "prompt_1": { "accuracy": 0.7757009345794392 }, "prompt_2": { "accuracy": 0.7757009345794392 }, "prompt_3": { "accuracy": 0.7757009345794392 }, "prompt_4": { "accuracy": 0.794392523364486 }, "prompt_5": { "accuracy": 0.8037383177570093 } }, "ph_eval": { "prompt_1": { "accuracy": 0.62, "category_acc": { "brand": 0.6, "demographics": 0.4, "biology": 0.8, "history": 0.5333333333333333, "literature": 0.3, "politics": 0.8, "culture": 0.9, "film": 0.6, "law": 0.7, "geography": 0.5 } }, "prompt_2": { "accuracy": 0.67, "category_acc": { "brand": 0.6, "demographics": 0.4, "biology": 0.8, "history": 0.6, "literature": 0.5, "politics": 0.8, "culture": 1.0, "film": 0.6, "law": 0.8, "geography": 0.5 } }, "prompt_3": { "accuracy": 0.61, "category_acc": { "brand": 0.1, "demographics": 0.4, "biology": 0.8, "history": 0.6, "literature": 0.5, "politics": 0.7, "culture": 1.0, "film": 0.7, "law": 0.5, "geography": 0.7 } }, "prompt_4": { "accuracy": 0.66, "category_acc": { "brand": 0.3, "demographics": 0.4, "biology": 0.8, "history": 0.7333333333333333, "literature": 0.4, "politics": 0.8, "culture": 1.0, "film": 0.6, "law": 0.7, "geography": 0.7 } }, "prompt_5": { "accuracy": 0.6, "category_acc": { "brand": 0.7, "demographics": 0.2, "biology": 0.7, "history": 0.5333333333333333, "literature": 0.4, "politics": 0.4, "culture": 0.9, "film": 0.7, "law": 0.6, "geography": 0.7 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.22080062628043398 }, "prompt_2": { "bleu_score": 0.19275817915230986 }, "prompt_3": { "bleu_score": 0.1991606048845971 }, "prompt_4": { "bleu_score": 0.2278437314247922 }, "prompt_5": { "bleu_score": 0.1650427174776864 } }, "indommlu": { "prompt_1": { "accuracy": 0.566526470391882, "category_acc": { "History": 0.5622489959839357, "Geography": 0.5653061224489796, "Lampungic": 0.3129251700680272, "Social science": 0.8163606010016694, "Balinese": 0.33121019108280253, "Makassarese": 0.3655913978494624, "Banjarese": 0.3263888888888889, "Chemistry": 0.39416058394160586, "Biology": 0.5834319526627219, "Science": 0.7471620227038184, "Christian religion": 0.7064676616915423, "Art": 0.6821963394342762, "Islam religion": 0.6714082503556188, "Hindu religion": 0.6, "Madurese": 0.31186440677966104, "Sport": 0.5675675675675675, "Indonesian language": 0.6422789539227896, "Physics": 0.4888888888888889, "Minangkabau culture": 0.457286432160804, "Dayak language": 0.3486238532110092, "Sociology": 0.5645161290322581, "Economy": 0.5963114754098361, "Sundanese": 0.428694900605013, "Javanese": 0.3780241935483871, "Civic education": 0.6723891273247496 } }, "prompt_2": { "accuracy": 0.5675278723546299, "category_acc": { "History": 0.5562248995983936, "Geography": 0.5551020408163265, "Lampungic": 0.3401360544217687, "Social science": 0.8163606010016694, "Balinese": 0.32696390658174096, "Makassarese": 0.3333333333333333, "Banjarese": 0.3541666666666667, "Chemistry": 0.39416058394160586, "Biology": 0.6, "Science": 0.737874097007224, "Christian religion": 0.736318407960199, "Art": 0.6871880199667221, "Islam religion": 0.6571834992887624, "Hindu religion": 0.6066666666666667, "Madurese": 0.31186440677966104, "Sport": 0.5608108108108109, "Indonesian language": 0.6438356164383562, "Physics": 0.4888888888888889, "Minangkabau culture": 0.45226130653266333, "Dayak language": 0.3394495412844037, "Sociology": 0.5645161290322581, "Economy": 0.5901639344262295, "Sundanese": 0.44165946413137425, "Javanese": 0.3860887096774194, "Civic education": 0.6666666666666666 } }, "prompt_3": { "accuracy": 0.5708658788971226, "category_acc": { "History": 0.5662650602409639, "Geography": 0.5795918367346938, "Lampungic": 0.3469387755102041, "Social science": 0.8130217028380634, "Balinese": 0.3184713375796178, "Makassarese": 0.34946236559139787, "Banjarese": 0.4097222222222222, "Chemistry": 0.37518248175182484, "Biology": 0.6047337278106509, "Science": 0.7420020639834881, "Christian religion": 0.7313432835820896, "Art": 0.6921797004991681, "Islam religion": 0.6856330014224751, "Hindu religion": 0.6, "Madurese": 0.3288135593220339, "Sport": 0.5540540540540541, "Indonesian language": 0.6463262764632628, "Physics": 0.49696969696969695, "Minangkabau culture": 0.47738693467336685, "Dayak language": 0.3486238532110092, "Sociology": 0.5645161290322581, "Economy": 0.6024590163934426, "Sundanese": 0.4347450302506482, "Javanese": 0.3780241935483871, "Civic education": 0.6652360515021459 } }, "prompt_4": { "accuracy": 0.5680619534014286, "category_acc": { "History": 0.5522088353413654, "Geography": 0.5857142857142857, "Lampungic": 0.3741496598639456, "Social science": 0.8146911519198664, "Balinese": 0.3057324840764331, "Makassarese": 0.3387096774193548, "Banjarese": 0.4444444444444444, "Chemistry": 0.38686131386861317, "Biology": 0.5964497041420118, "Science": 0.7420020639834881, "Christian religion": 0.736318407960199, "Art": 0.6805324459234608, "Islam religion": 0.6913229018492176, "Hindu religion": 0.6, "Madurese": 0.3016949152542373, "Sport": 0.5472972972972973, "Indonesian language": 0.6438356164383562, "Physics": 0.47878787878787876, "Minangkabau culture": 0.46733668341708545, "Dayak language": 0.3394495412844037, "Sociology": 0.5625, "Economy": 0.5860655737704918, "Sundanese": 0.42955920484010374, "Javanese": 0.3800403225806452, "Civic education": 0.6695278969957081 } }, "prompt_5": { "accuracy": 0.5594498965217972, "category_acc": { "History": 0.5522088353413654, "Geography": 0.5693877551020409, "Lampungic": 0.32653061224489793, "Social science": 0.7996661101836394, "Balinese": 0.32908704883227174, "Makassarese": 0.3924731182795699, "Banjarese": 0.3611111111111111, "Chemistry": 0.35474452554744523, "Biology": 0.5763313609467455, "Science": 0.7533539731682146, "Christian religion": 0.736318407960199, "Art": 0.6888519134775375, "Islam religion": 0.6671408250355618, "Hindu religion": 0.54, "Madurese": 0.29491525423728815, "Sport": 0.5472972972972973, "Indonesian language": 0.6338729763387297, "Physics": 0.48484848484848486, "Minangkabau culture": 0.4623115577889447, "Dayak language": 0.3119266055045872, "Sociology": 0.5544354838709677, "Economy": 0.5799180327868853, "Sundanese": 0.4338807260155575, "Javanese": 0.3558467741935484, "Civic education": 0.6638054363376252 } } }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": { "bleu_score": 0.3339780933221536 }, "prompt_3": { "bleu_score": 0.33747129268713477 }, "prompt_4": { "bleu_score": 0.3422115305107942 }, "prompt_5": { "bleu_score": 0.3277128266213432 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.2601528872340481 }, "prompt_2": { "bleu_score": 0.2723729015879098 }, "prompt_3": { "bleu_score": 0.26978902531768745 }, "prompt_4": { "bleu_score": 0.27433289324803795 }, "prompt_5": { "bleu_score": 0.25871960099169916 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.23010635681854288 }, "prompt_2": { "bleu_score": 0.22391087370987364 }, "prompt_3": { "bleu_score": 0.22973251292515848 }, "prompt_4": { "bleu_score": 0.23198679815051781 }, "prompt_5": { "bleu_score": 0.22891320560883185 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.33716412207415664 }, "prompt_2": { "bleu_score": 0.343058023779828 }, "prompt_3": { "bleu_score": 0.34339490478084983 }, "prompt_4": { "bleu_score": 0.3493321057945565 }, "prompt_5": { "bleu_score": 0.33508961110505703 } }, "mmlu": { "prompt_1": { "accuracy": 0.6896149358226371 }, "prompt_2": { "accuracy": 0.6931155192532089 }, "prompt_3": { "accuracy": 0.7036172695449242 }, "prompt_4": { "accuracy": 0.6861143523920653 }, "prompt_5": { "accuracy": 0.6569428238039673 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.6792277440114408, "category_acc": { "high_school_european_history": 0.7926829268292683, "business_ethics": 0.7676767676767676, "clinical_knowledge": 0.7424242424242424, "medical_genetics": 0.7474747474747475, "high_school_us_history": 0.8669950738916257, "high_school_physics": 0.4, "high_school_world_history": 0.8771186440677966, "virology": 0.503030303030303, "high_school_microeconomics": 0.7721518987341772, "econometrics": 0.5132743362831859, "college_computer_science": 0.5858585858585859, "high_school_biology": 0.8220064724919094, "abstract_algebra": 0.26262626262626265, "professional_accounting": 0.5765124555160143, "philosophy": 0.6580645161290323, "professional_medicine": 0.7416974169741697, "nutrition": 0.7377049180327869, "global_facts": 0.42424242424242425, "machine_learning": 0.4864864864864865, "security_studies": 0.7008196721311475, "public_relations": 0.6330275229357798, "professional_psychology": 0.7152209492635024, "prehistory": 0.7492260061919505, "anatomy": 0.6417910447761194, "human_sexuality": 0.7461538461538462, "college_medicine": 0.6686046511627907, "high_school_government_and_politics": 0.9479166666666666, "college_chemistry": 0.48484848484848486, "logical_fallacies": 0.7592592592592593, "high_school_geography": 0.8578680203045685, "elementary_mathematics": 0.3103448275862069, "human_aging": 0.7072072072072072, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.8492647058823529, "formal_logic": 0.504, "high_school_statistics": 0.5209302325581395, "international_law": 0.7833333333333333, "high_school_mathematics": 0.275092936802974, "high_school_computer_science": 0.7474747474747475, "conceptual_physics": 0.6581196581196581, "miscellaneous": 0.9181585677749361, "high_school_chemistry": 0.47029702970297027, "marketing": 0.8798283261802575, "professional_law": 0.6046966731898239, "management": 0.7941176470588235, "college_physics": 0.39603960396039606, "jurisprudence": 0.794392523364486, "world_religions": 0.8235294117647058, "sociology": 0.795, "us_foreign_policy": 0.7777777777777778, "high_school_macroeconomics": 0.6529562982005142, "computer_security": 0.7676767676767676, "moral_scenarios": 0.6532438478747203, "moral_disputes": 0.5768115942028985, "electrical_engineering": 0.5486111111111112, "astronomy": 0.8079470198675497, "college_biology": 0.8041958041958042 } }, "prompt_2": { "accuracy": 0.7095459420808009, "category_acc": { "high_school_european_history": 0.8109756097560976, "business_ethics": 0.7373737373737373, "clinical_knowledge": 0.7613636363636364, "medical_genetics": 0.797979797979798, "high_school_us_history": 0.8620689655172413, "high_school_physics": 0.44, "high_school_world_history": 0.864406779661017, "virology": 0.509090909090909, "high_school_microeconomics": 0.8143459915611815, "econometrics": 0.5132743362831859, "college_computer_science": 0.5959595959595959, "high_school_biology": 0.8446601941747572, "abstract_algebra": 0.26262626262626265, "professional_accounting": 0.6192170818505338, "philosophy": 0.7258064516129032, "professional_medicine": 0.8191881918819188, "nutrition": 0.7770491803278688, "global_facts": 0.42424242424242425, "machine_learning": 0.5225225225225225, "security_studies": 0.7540983606557377, "public_relations": 0.6972477064220184, "professional_psychology": 0.7512274959083469, "prehistory": 0.8018575851393189, "anatomy": 0.6492537313432836, "human_sexuality": 0.7615384615384615, "college_medicine": 0.7267441860465116, "high_school_government_and_politics": 0.9375, "college_chemistry": 0.43434343434343436, "logical_fallacies": 0.7777777777777778, "high_school_geography": 0.8781725888324873, "elementary_mathematics": 0.41114058355437666, "human_aging": 0.6936936936936937, "college_mathematics": 0.36363636363636365, "high_school_psychology": 0.8823529411764706, "formal_logic": 0.544, "high_school_statistics": 0.5488372093023256, "international_law": 0.8166666666666667, "high_school_mathematics": 0.26394052044609667, "high_school_computer_science": 0.7474747474747475, "conceptual_physics": 0.717948717948718, "miscellaneous": 0.9245524296675192, "high_school_chemistry": 0.5495049504950495, "marketing": 0.8927038626609443, "professional_law": 0.6118721461187214, "management": 0.7843137254901961, "college_physics": 0.5247524752475248, "jurisprudence": 0.8130841121495327, "world_religions": 0.8352941176470589, "sociology": 0.815, "us_foreign_policy": 0.8282828282828283, "high_school_macroeconomics": 0.6786632390745502, "computer_security": 0.7676767676767676, "moral_scenarios": 0.6923937360178971, "moral_disputes": 0.6521739130434783, "electrical_engineering": 0.6944444444444444, "astronomy": 0.8410596026490066, "college_biology": 0.8531468531468531 } }, "prompt_3": { "accuracy": 0.7092599213442975, "category_acc": { "high_school_european_history": 0.823170731707317, "business_ethics": 0.7575757575757576, "clinical_knowledge": 0.7878787878787878, "medical_genetics": 0.7777777777777778, "high_school_us_history": 0.8817733990147784, "high_school_physics": 0.46, "high_school_world_history": 0.8771186440677966, "virology": 0.5212121212121212, "high_school_microeconomics": 0.8312236286919831, "econometrics": 0.5132743362831859, "college_computer_science": 0.5757575757575758, "high_school_biology": 0.8511326860841424, "abstract_algebra": 0.3939393939393939, "professional_accounting": 0.5587188612099644, "philosophy": 0.7483870967741936, "professional_medicine": 0.7822878228782287, "nutrition": 0.7770491803278688, "global_facts": 0.48484848484848486, "machine_learning": 0.5225225225225225, "security_studies": 0.7581967213114754, "public_relations": 0.7339449541284404, "professional_psychology": 0.7659574468085106, "prehistory": 0.8080495356037152, "anatomy": 0.6940298507462687, "human_sexuality": 0.7615384615384615, "college_medicine": 0.7325581395348837, "high_school_government_and_politics": 0.9583333333333334, "college_chemistry": 0.5353535353535354, "logical_fallacies": 0.7839506172839507, "high_school_geography": 0.8934010152284264, "elementary_mathematics": 0.3819628647214854, "human_aging": 0.7072072072072072, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.875, "formal_logic": 0.544, "high_school_statistics": 0.5581395348837209, "international_law": 0.8333333333333334, "high_school_mathematics": 0.23048327137546468, "high_school_computer_science": 0.7676767676767676, "conceptual_physics": 0.717948717948718, "miscellaneous": 0.921994884910486, "high_school_chemistry": 0.5544554455445545, "marketing": 0.8969957081545065, "professional_law": 0.5962165688193085, "management": 0.7941176470588235, "college_physics": 0.48514851485148514, "jurisprudence": 0.8130841121495327, "world_religions": 0.8470588235294118, "sociology": 0.84, "us_foreign_policy": 0.8585858585858586, "high_school_macroeconomics": 0.7043701799485861, "computer_security": 0.7575757575757576, "moral_scenarios": 0.639821029082774, "moral_disputes": 0.6985507246376812, "electrical_engineering": 0.6319444444444444, "astronomy": 0.8079470198675497, "college_biology": 0.8601398601398601 } }, "prompt_4": { "accuracy": 0.6888809438684305, "category_acc": { "high_school_european_history": 0.7987804878048781, "business_ethics": 0.7676767676767676, "clinical_knowledge": 0.7727272727272727, "medical_genetics": 0.7777777777777778, "high_school_us_history": 0.8817733990147784, "high_school_physics": 0.41333333333333333, "high_school_world_history": 0.8813559322033898, "virology": 0.4484848484848485, "high_school_microeconomics": 0.7932489451476793, "econometrics": 0.5132743362831859, "college_computer_science": 0.5454545454545454, "high_school_biology": 0.8446601941747572, "abstract_algebra": 0.1919191919191919, "professional_accounting": 0.5444839857651246, "philosophy": 0.7129032258064516, "professional_medicine": 0.6752767527675276, "nutrition": 0.7475409836065574, "global_facts": 0.45454545454545453, "machine_learning": 0.5045045045045045, "security_studies": 0.7459016393442623, "public_relations": 0.6513761467889908, "professional_psychology": 0.7364975450081833, "prehistory": 0.7925696594427245, "anatomy": 0.6940298507462687, "human_sexuality": 0.7384615384615385, "college_medicine": 0.7034883720930233, "high_school_government_and_politics": 0.9583333333333334, "college_chemistry": 0.4444444444444444, "logical_fallacies": 0.7469135802469136, "high_school_geography": 0.8781725888324873, "elementary_mathematics": 0.35278514588859416, "human_aging": 0.7117117117117117, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.8547794117647058, "formal_logic": 0.512, "high_school_statistics": 0.5116279069767442, "international_law": 0.8083333333333333, "high_school_mathematics": 0.21933085501858737, "high_school_computer_science": 0.7070707070707071, "conceptual_physics": 0.7051282051282052, "miscellaneous": 0.9207161125319693, "high_school_chemistry": 0.504950495049505, "marketing": 0.8798283261802575, "professional_law": 0.5981735159817352, "management": 0.7843137254901961, "college_physics": 0.43564356435643564, "jurisprudence": 0.7850467289719626, "world_religions": 0.8117647058823529, "sociology": 0.82, "us_foreign_policy": 0.8585858585858586, "high_school_macroeconomics": 0.6606683804627249, "computer_security": 0.7676767676767676, "moral_scenarios": 0.685682326621924, "moral_disputes": 0.6289855072463768, "electrical_engineering": 0.5625, "astronomy": 0.8079470198675497, "college_biology": 0.7902097902097902 } }, "prompt_5": { "accuracy": 0.6712191633893457, "category_acc": { "high_school_european_history": 0.8109756097560976, "business_ethics": 0.7070707070707071, "clinical_knowledge": 0.7348484848484849, "medical_genetics": 0.7676767676767676, "high_school_us_history": 0.8768472906403941, "high_school_physics": 0.42, "high_school_world_history": 0.8686440677966102, "virology": 0.509090909090909, "high_school_microeconomics": 0.7890295358649789, "econometrics": 0.46017699115044247, "college_computer_science": 0.5858585858585859, "high_school_biology": 0.8025889967637541, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.5409252669039146, "philosophy": 0.6516129032258065, "professional_medicine": 0.6531365313653137, "nutrition": 0.7508196721311475, "global_facts": 0.42424242424242425, "machine_learning": 0.4594594594594595, "security_studies": 0.7377049180327869, "public_relations": 0.6513761467889908, "professional_psychology": 0.6808510638297872, "prehistory": 0.7708978328173375, "anatomy": 0.6791044776119403, "human_sexuality": 0.7076923076923077, "college_medicine": 0.6744186046511628, "high_school_government_and_politics": 0.9479166666666666, "college_chemistry": 0.48484848484848486, "logical_fallacies": 0.6790123456790124, "high_school_geography": 0.8578680203045685, "elementary_mathematics": 0.33421750663129973, "human_aging": 0.7117117117117117, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.8492647058823529, "formal_logic": 0.432, "high_school_statistics": 0.5302325581395348, "international_law": 0.825, "high_school_mathematics": 0.19330855018587362, "high_school_computer_science": 0.7373737373737373, "conceptual_physics": 0.6623931623931624, "miscellaneous": 0.9117647058823529, "high_school_chemistry": 0.504950495049505, "marketing": 0.8884120171673819, "professional_law": 0.593607305936073, "management": 0.7647058823529411, "college_physics": 0.44554455445544555, "jurisprudence": 0.7476635514018691, "world_religions": 0.8176470588235294, "sociology": 0.795, "us_foreign_policy": 0.8383838383838383, "high_school_macroeconomics": 0.6349614395886889, "computer_security": 0.7575757575757576, "moral_scenarios": 0.6297539149888143, "moral_disputes": 0.5652173913043478, "electrical_engineering": 0.5694444444444444, "astronomy": 0.8211920529801324, "college_biology": 0.7762237762237763 } } }, "c_eval": { "prompt_1": { "accuracy": 0.7971768202080238 }, "prompt_2": { "accuracy": 0.7659732540861813 }, "prompt_3": { "accuracy": 0.7585438335809807 }, "prompt_4": { "accuracy": 0.638187221396731 }, "prompt_5": { "accuracy": 0.687964338781575 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.8119551681195517, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.8333333333333334, "computer_architecture": 0.8846153846153846, "college_programming": 0.8333333333333334, "college_physics": 0.75, "college_chemistry": 0.6206896551724138, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.7142857142857143, "metrology_engineer": 0.896551724137931, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.7916666666666666, "high_school_chemistry": 0.7916666666666666, "high_school_biology": 0.9166666666666666, "middle_school_mathematics": 0.6666666666666666, "middle_school_biology": 0.9615384615384616, "middle_school_physics": 0.9166666666666666, "middle_school_chemistry": 1.0, "veterinary_medicine": 0.8571428571428571, "college_economics": 0.7166666666666667, "business_administration": 0.7631578947368421, "marxism": 0.9166666666666666, "mao_zedong_thought": 0.9655172413793104, "education_science": 0.8823529411764706, "teacher_qualification": 0.9183673469387755, "high_school_politics": 1.0, "high_school_geography": 0.9166666666666666, "middle_school_politics": 1.0, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.9285714285714286, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.7777777777777778, "law": 0.8275862068965517, "chinese_language_and_literature": 0.7142857142857143, "art_studies": 0.8947368421052632, "professional_tour_guide": 0.9411764705882353, "legal_professional": 0.6785714285714286, "high_school_chinese": 0.875, "high_school_history": 1.0, "middle_school_history": 0.9259259259259259, "civil_servant": 0.75, "sports_science": 0.875, "plant_protection": 0.8888888888888888, "basic_medicine": 0.9166666666666666, "clinical_medicine": 0.8148148148148148, "urban_and_rural_planner": 0.8235294117647058, "accountant": 0.8518518518518519, "fire_engineer": 0.8055555555555556, "environmental_impact_assessment_engineer": 0.8055555555555556, "tax_accountant": 0.8703703703703703, "physician": 0.8518518518518519 } }, "prompt_2": { "accuracy": 0.7845579078455791, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.7916666666666666, "computer_architecture": 0.6923076923076923, "college_programming": 0.8333333333333334, "college_physics": 0.5833333333333334, "college_chemistry": 0.6206896551724138, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.7380952380952381, "metrology_engineer": 0.8275862068965517, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.7916666666666666, "high_school_chemistry": 0.7083333333333334, "high_school_biology": 0.7916666666666666, "middle_school_mathematics": 0.4166666666666667, "middle_school_biology": 0.9615384615384616, "middle_school_physics": 0.9166666666666666, "middle_school_chemistry": 0.96, "veterinary_medicine": 0.8571428571428571, "college_economics": 0.6666666666666666, "business_administration": 0.7894736842105263, "marxism": 0.9166666666666666, "mao_zedong_thought": 0.9310344827586207, "education_science": 0.8823529411764706, "teacher_qualification": 0.9387755102040817, "high_school_politics": 1.0, "high_school_geography": 0.9166666666666666, "middle_school_politics": 1.0, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.8928571428571429, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.7407407407407407, "law": 0.7931034482758621, "chinese_language_and_literature": 0.75, "art_studies": 0.8947368421052632, "professional_tour_guide": 0.9411764705882353, "legal_professional": 0.6428571428571429, "high_school_chinese": 0.8333333333333334, "high_school_history": 1.0, "middle_school_history": 0.9629629629629629, "civil_servant": 0.7884615384615384, "sports_science": 0.9166666666666666, "plant_protection": 0.7777777777777778, "basic_medicine": 0.9166666666666666, "clinical_medicine": 0.8148148148148148, "urban_and_rural_planner": 0.803921568627451, "accountant": 0.8333333333333334, "fire_engineer": 0.7222222222222222, "environmental_impact_assessment_engineer": 0.7777777777777778, "tax_accountant": 0.8333333333333334, "physician": 0.8333333333333334 } }, "prompt_3": { "accuracy": 0.7914072229140723, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.875, "computer_architecture": 0.8076923076923077, "college_programming": 0.8095238095238095, "college_physics": 0.625, "college_chemistry": 0.6551724137931034, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.7380952380952381, "metrology_engineer": 0.8620689655172413, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.75, "high_school_chemistry": 0.6666666666666666, "high_school_biology": 0.8333333333333334, "middle_school_mathematics": 0.4166666666666667, "middle_school_biology": 0.9230769230769231, "middle_school_physics": 0.9166666666666666, "middle_school_chemistry": 0.96, "veterinary_medicine": 0.8571428571428571, "college_economics": 0.6666666666666666, "business_administration": 0.6842105263157895, "marxism": 0.9166666666666666, "mao_zedong_thought": 0.9310344827586207, "education_science": 0.8529411764705882, "teacher_qualification": 0.9387755102040817, "high_school_politics": 1.0, "high_school_geography": 0.9166666666666666, "middle_school_politics": 1.0, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.8928571428571429, "ideological_and_moral_cultivation": 0.9166666666666666, "logic": 0.7407407407407407, "law": 0.8275862068965517, "chinese_language_and_literature": 0.6785714285714286, "art_studies": 0.8947368421052632, "professional_tour_guide": 0.9411764705882353, "legal_professional": 0.6428571428571429, "high_school_chinese": 0.875, "high_school_history": 1.0, "middle_school_history": 0.9629629629629629, "civil_servant": 0.75, "sports_science": 0.875, "plant_protection": 0.8888888888888888, "basic_medicine": 0.9166666666666666, "clinical_medicine": 0.8518518518518519, "urban_and_rural_planner": 0.8235294117647058, "accountant": 0.8333333333333334, "fire_engineer": 0.7777777777777778, "environmental_impact_assessment_engineer": 0.8055555555555556, "tax_accountant": 0.8333333333333334, "physician": 0.8518518518518519 } }, "prompt_4": { "accuracy": 0.6376089663760897, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.7083333333333334, "computer_architecture": 0.6923076923076923, "college_programming": 0.7380952380952381, "college_physics": 0.20833333333333334, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.7241379310344828, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.5, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.7083333333333334, "middle_school_mathematics": 0.375, "middle_school_biology": 0.9230769230769231, "middle_school_physics": 0.7083333333333334, "middle_school_chemistry": 0.72, "veterinary_medicine": 0.5714285714285714, "college_economics": 0.36666666666666664, "business_administration": 0.5263157894736842, "marxism": 0.7083333333333334, "mao_zedong_thought": 0.9655172413793104, "education_science": 0.7352941176470589, "teacher_qualification": 0.9387755102040817, "high_school_politics": 0.7916666666666666, "high_school_geography": 0.75, "middle_school_politics": 0.9615384615384616, "middle_school_geography": 0.8823529411764706, "modern_chinese_history": 0.8928571428571429, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.6666666666666666, "law": 0.7241379310344828, "chinese_language_and_literature": 0.75, "art_studies": 0.8421052631578947, "professional_tour_guide": 0.9411764705882353, "legal_professional": 0.5357142857142857, "high_school_chinese": 0.6666666666666666, "high_school_history": 0.84, "middle_school_history": 0.9259259259259259, "civil_servant": 0.6538461538461539, "sports_science": 0.75, "plant_protection": 0.4074074074074074, "basic_medicine": 0.7916666666666666, "clinical_medicine": 0.6296296296296297, "urban_and_rural_planner": 0.5098039215686274, "accountant": 0.6481481481481481, "fire_engineer": 0.4444444444444444, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.5555555555555556, "physician": 0.6481481481481481 } }, "prompt_5": { "accuracy": 0.6836861768368617, "category_acc": { "computer_network": 0.5833333333333334, "operating_system": 0.5833333333333334, "computer_architecture": 0.7692307692307693, "college_programming": 0.6190476190476191, "college_physics": 0.4583333333333333, "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.5714285714285714, "metrology_engineer": 0.8620689655172413, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.7083333333333334, "high_school_chemistry": 0.5416666666666666, "high_school_biology": 0.7916666666666666, "middle_school_mathematics": 0.5416666666666666, "middle_school_biology": 0.7692307692307693, "middle_school_physics": 0.7916666666666666, "middle_school_chemistry": 0.88, "veterinary_medicine": 0.6071428571428571, "college_economics": 0.5833333333333334, "business_administration": 0.6052631578947368, "marxism": 0.75, "mao_zedong_thought": 0.8620689655172413, "education_science": 0.7058823529411765, "teacher_qualification": 0.9183673469387755, "high_school_politics": 1.0, "high_school_geography": 0.9166666666666666, "middle_school_politics": 0.9230769230769231, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.6785714285714286, "ideological_and_moral_cultivation": 0.9166666666666666, "logic": 0.6296296296296297, "law": 0.6206896551724138, "chinese_language_and_literature": 0.75, "art_studies": 0.8421052631578947, "professional_tour_guide": 0.9411764705882353, "legal_professional": 0.4642857142857143, "high_school_chinese": 0.8333333333333334, "high_school_history": 0.84, "middle_school_history": 0.9259259259259259, "civil_servant": 0.5384615384615384, "sports_science": 0.75, "plant_protection": 0.6296296296296297, "basic_medicine": 0.75, "clinical_medicine": 0.7407407407407407, "urban_and_rural_planner": 0.7450980392156863, "accountant": 0.7592592592592593, "fire_engineer": 0.6944444444444444, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.6481481481481481, "physician": 0.6666666666666666 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.7849462365591398 }, "prompt_2": { "accuracy": 0.7813620071684588 }, "prompt_3": { "accuracy": 0.7885304659498208 }, "prompt_4": { "accuracy": 0.6881720430107527 }, "prompt_5": { "accuracy": 0.6953405017921147 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.8172163702296668, "category_acc": { "agronomy": 0.7633136094674556, "anatomy": 0.8716216216216216, "ancient_chinese": 0.5670731707317073, "arts": 0.9625, "astronomy": 0.5878787878787879, "business_ethics": 0.7177033492822966, "chinese_civil_service_exam": 0.76875, "chinese_driving_rule": 0.9694656488549618, "chinese_food_culture": 0.8455882352941176, "chinese_foreign_policy": 0.9345794392523364, "chinese_history": 0.9411764705882353, "chinese_literature": 0.7990196078431373, "chinese_teacher_qualification": 0.9217877094972067, "clinical_knowledge": 0.8481012658227848, "college_actuarial_science": 0.2830188679245283, "college_education": 0.9252336448598131, "college_engineering_hydrology": 0.8113207547169812, "college_law": 0.8055555555555556, "college_mathematics": 0.44761904761904764, "college_medical_statistics": 0.7169811320754716, "college_medicine": 0.8681318681318682, "computer_science": 0.8725490196078431, "computer_security": 0.9122807017543859, "conceptual_physics": 0.9047619047619048, "construction_project_management": 0.7266187050359713, "economics": 0.8301886792452831, "education": 0.7852760736196319, "electrical_engineering": 0.813953488372093, "elementary_chinese": 0.7936507936507936, "elementary_commonsense": 0.797979797979798, "elementary_information_and_technology": 0.9369747899159664, "elementary_mathematics": 0.48695652173913045, "ethnology": 0.8444444444444444, "food_science": 0.7412587412587412, "genetics": 0.7045454545454546, "global_facts": 0.8523489932885906, "high_school_biology": 0.9289940828402367, "high_school_chemistry": 0.8484848484848485, "high_school_geography": 0.923728813559322, "high_school_mathematics": 0.4146341463414634, "high_school_physics": 0.7909090909090909, "high_school_politics": 0.9230769230769231, "human_sexuality": 0.8015873015873016, "international_law": 0.8054054054054054, "journalism": 0.7848837209302325, "jurisprudence": 0.8613138686131386, "legal_and_moral_basis": 0.9766355140186916, "logical": 0.7886178861788617, "machine_learning": 0.7213114754098361, "management": 0.9095238095238095, "marketing": 0.8944444444444445, "marxist_theory": 0.9417989417989417, "modern_chinese": 0.6810344827586207, "nutrition": 0.7931034482758621, "philosophy": 0.8476190476190476, "professional_accounting": 0.9314285714285714, "professional_law": 0.7867298578199052, "professional_medicine": 0.8031914893617021, "professional_psychology": 0.9008620689655172, "public_relations": 0.7873563218390804, "security_study": 0.9259259259259259, "sociology": 0.8097345132743363, "sports_science": 0.7818181818181819, "traditional_chinese_medicine": 0.8810810810810811, "virology": 0.8579881656804734, "world_history": 0.9254658385093167, "world_religions": 0.86875 } }, "prompt_2": { "accuracy": 0.7912277672250043, "category_acc": { "agronomy": 0.7218934911242604, "anatomy": 0.8716216216216216, "ancient_chinese": 0.5609756097560976, "arts": 0.9625, "astronomy": 0.5272727272727272, "business_ethics": 0.6889952153110048, "chinese_civil_service_exam": 0.75625, "chinese_driving_rule": 0.9465648854961832, "chinese_food_culture": 0.8161764705882353, "chinese_foreign_policy": 0.8878504672897196, "chinese_history": 0.913312693498452, "chinese_literature": 0.8088235294117647, "chinese_teacher_qualification": 0.8938547486033519, "clinical_knowledge": 0.8227848101265823, "college_actuarial_science": 0.3113207547169811, "college_education": 0.9252336448598131, "college_engineering_hydrology": 0.7830188679245284, "college_law": 0.7592592592592593, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.5660377358490566, "college_medicine": 0.8498168498168498, "computer_science": 0.8431372549019608, "computer_security": 0.9064327485380117, "conceptual_physics": 0.8843537414965986, "construction_project_management": 0.697841726618705, "economics": 0.7610062893081762, "education": 0.7668711656441718, "electrical_engineering": 0.7558139534883721, "elementary_chinese": 0.7777777777777778, "elementary_commonsense": 0.797979797979798, "elementary_information_and_technology": 0.9285714285714286, "elementary_mathematics": 0.41304347826086957, "ethnology": 0.837037037037037, "food_science": 0.7132867132867133, "genetics": 0.6704545454545454, "global_facts": 0.87248322147651, "high_school_biology": 0.9053254437869822, "high_school_chemistry": 0.7803030303030303, "high_school_geography": 0.8898305084745762, "high_school_mathematics": 0.38414634146341464, "high_school_physics": 0.7, "high_school_politics": 0.9020979020979021, "human_sexuality": 0.7142857142857143, "international_law": 0.7783783783783784, "journalism": 0.7732558139534884, "jurisprudence": 0.8491484184914841, "legal_and_moral_basis": 0.9813084112149533, "logical": 0.7479674796747967, "machine_learning": 0.6557377049180327, "management": 0.9095238095238095, "marketing": 0.8833333333333333, "marxist_theory": 0.9259259259259259, "modern_chinese": 0.6810344827586207, "nutrition": 0.7931034482758621, "philosophy": 0.8285714285714286, "professional_accounting": 0.8857142857142857, "professional_law": 0.7298578199052133, "professional_medicine": 0.8058510638297872, "professional_psychology": 0.8793103448275862, "public_relations": 0.7701149425287356, "security_study": 0.8740740740740741, "sociology": 0.7876106194690266, "sports_science": 0.806060606060606, "traditional_chinese_medicine": 0.8432432432432433, "virology": 0.8224852071005917, "world_history": 0.9006211180124224, "world_religions": 0.83125 } }, "prompt_3": { "accuracy": 0.7882058366430669, "category_acc": { "agronomy": 0.7218934911242604, "anatomy": 0.8648648648648649, "ancient_chinese": 0.5304878048780488, "arts": 0.9625, "astronomy": 0.5393939393939394, "business_ethics": 0.6842105263157895, "chinese_civil_service_exam": 0.73125, "chinese_driving_rule": 0.9236641221374046, "chinese_food_culture": 0.8161764705882353, "chinese_foreign_policy": 0.8598130841121495, "chinese_history": 0.9164086687306502, "chinese_literature": 0.7745098039215687, "chinese_teacher_qualification": 0.8994413407821229, "clinical_knowledge": 0.7974683544303798, "college_actuarial_science": 0.29245283018867924, "college_education": 0.9345794392523364, "college_engineering_hydrology": 0.7830188679245284, "college_law": 0.75, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.6415094339622641, "college_medicine": 0.8461538461538461, "computer_science": 0.8627450980392157, "computer_security": 0.9181286549707602, "conceptual_physics": 0.8639455782312925, "construction_project_management": 0.697841726618705, "economics": 0.7861635220125787, "education": 0.7607361963190185, "electrical_engineering": 0.7616279069767442, "elementary_chinese": 0.7777777777777778, "elementary_commonsense": 0.797979797979798, "elementary_information_and_technology": 0.9369747899159664, "elementary_mathematics": 0.44782608695652176, "ethnology": 0.8518518518518519, "food_science": 0.7132867132867133, "genetics": 0.6534090909090909, "global_facts": 0.8456375838926175, "high_school_biology": 0.8816568047337278, "high_school_chemistry": 0.7878787878787878, "high_school_geography": 0.8898305084745762, "high_school_mathematics": 0.29878048780487804, "high_school_physics": 0.7, "high_school_politics": 0.8811188811188811, "human_sexuality": 0.7380952380952381, "international_law": 0.8, "journalism": 0.7616279069767442, "jurisprudence": 0.8345498783454988, "legal_and_moral_basis": 0.9766355140186916, "logical": 0.7560975609756098, "machine_learning": 0.6885245901639344, "management": 0.8904761904761904, "marketing": 0.8777777777777778, "marxist_theory": 0.9153439153439153, "modern_chinese": 0.6810344827586207, "nutrition": 0.7862068965517242, "philosophy": 0.8380952380952381, "professional_accounting": 0.8971428571428571, "professional_law": 0.7440758293838863, "professional_medicine": 0.800531914893617, "professional_psychology": 0.875, "public_relations": 0.7701149425287356, "security_study": 0.9037037037037037, "sociology": 0.7920353982300885, "sports_science": 0.7878787878787878, "traditional_chinese_medicine": 0.827027027027027, "virology": 0.8165680473372781, "world_history": 0.906832298136646, "world_religions": 0.825 } }, "prompt_4": -1, "prompt_5": { "accuracy": 0.7184424106371956, "category_acc": { "agronomy": 0.6568047337278107, "anatomy": 0.7972972972972973, "ancient_chinese": 0.5060975609756098, "arts": 0.9375, "astronomy": 0.47878787878787876, "business_ethics": 0.6220095693779905, "chinese_civil_service_exam": 0.68125, "chinese_driving_rule": 0.8396946564885496, "chinese_food_culture": 0.8014705882352942, "chinese_foreign_policy": 0.8037383177570093, "chinese_history": 0.8080495356037152, "chinese_literature": 0.7401960784313726, "chinese_teacher_qualification": 0.8379888268156425, "clinical_knowledge": 0.7046413502109705, "college_actuarial_science": 0.3018867924528302, "college_education": 0.8691588785046729, "college_engineering_hydrology": 0.7358490566037735, "college_law": 0.6203703703703703, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.4716981132075472, "college_medicine": 0.7545787545787546, "computer_science": 0.803921568627451, "computer_security": 0.7543859649122807, "conceptual_physics": 0.8027210884353742, "construction_project_management": 0.60431654676259, "economics": 0.6729559748427673, "education": 0.7177914110429447, "electrical_engineering": 0.6627906976744186, "elementary_chinese": 0.7619047619047619, "elementary_commonsense": 0.7777777777777778, "elementary_information_and_technology": 0.8739495798319328, "elementary_mathematics": 0.391304347826087, "ethnology": 0.8148148148148148, "food_science": 0.6783216783216783, "genetics": 0.5284090909090909, "global_facts": 0.785234899328859, "high_school_biology": 0.8224852071005917, "high_school_chemistry": 0.7045454545454546, "high_school_geography": 0.788135593220339, "high_school_mathematics": 0.2926829268292683, "high_school_physics": 0.5909090909090909, "high_school_politics": 0.7762237762237763, "human_sexuality": 0.6190476190476191, "international_law": 0.6972972972972973, "journalism": 0.7325581395348837, "jurisprudence": 0.7980535279805353, "legal_and_moral_basis": 0.9392523364485982, "logical": 0.6747967479674797, "machine_learning": 0.5409836065573771, "management": 0.8, "marketing": 0.8611111111111112, "marxist_theory": 0.8888888888888888, "modern_chinese": 0.6551724137931034, "nutrition": 0.7034482758620689, "philosophy": 0.7619047619047619, "professional_accounting": 0.8285714285714286, "professional_law": 0.6255924170616114, "professional_medicine": 0.6728723404255319, "professional_psychology": 0.8318965517241379, "public_relations": 0.7126436781609196, "security_study": 0.7555555555555555, "sociology": 0.7522123893805309, "sports_science": 0.7151515151515152, "traditional_chinese_medicine": 0.7297297297297297, "virology": 0.727810650887574, "world_history": 0.7763975155279503, "world_religions": 0.8 } } }, "zbench": { "prompt_1": { "accuracy": 0.6060606060606061 }, "prompt_2": { "accuracy": 0.5757575757575758 }, "prompt_3": { "accuracy": 0.6363636363636364 }, "prompt_4": { "accuracy": 0.6060606060606061 }, "prompt_5": { "accuracy": 0.5454545454545454 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6909090909090909 }, "prompt_2": { "accuracy": 0.6795454545454546 }, "prompt_3": { "accuracy": 0.6727272727272727 }, "prompt_4": { "accuracy": 0.6272727272727273 }, "prompt_5": { "accuracy": 0.6454545454545455 } }, "ocnli": { "prompt_1": { "accuracy": 0.7935593220338983 }, "prompt_2": { "accuracy": 0.7894915254237288 }, "prompt_3": { "accuracy": 0.7813559322033898 }, "prompt_4": { "accuracy": 0.7515254237288136 }, "prompt_5": { "accuracy": 0.7549152542372881 } }, "c3": { "prompt_1": { "accuracy": 0.9554973821989529 }, "prompt_2": { "accuracy": 0.9390426327599103 }, "prompt_3": { "accuracy": 0.9416604338070307 }, "prompt_4": { "accuracy": 0.9412864622288706 }, "prompt_5": { "accuracy": 0.9330590875093493 } }, "dream": { "prompt_1": { "accuracy": 0.9333659970602646 }, "prompt_2": { "accuracy": 0.9397354238118569 }, "prompt_3": { "accuracy": 0.9387555120039196 }, "prompt_4": { "accuracy": 0.9343459088682019 }, "prompt_5": { "accuracy": 0.9377756001959824 } }, "samsum": { "prompt_1": { "rouge1": 0.3547401935449302, "rouge2": 0.11498075200152216, "rougeL": 0.2762344758589933, "avg_rouge": 0.24865180713514856 }, "prompt_2": { "rouge1": 0.3754300349232909, "rouge2": 0.13153300324103895, "rougeL": 0.29602806455697206, "avg_rouge": 0.2676637009071006 }, "prompt_3": { "rouge1": 0.36216209933720844, "rouge2": 0.12053893919079382, "rougeL": 0.2845717345450297, "avg_rouge": 0.255757591024344 }, "prompt_4": { "rouge1": 0.36173972133072, "rouge2": 0.12054748656750379, "rougeL": 0.28262952022661275, "avg_rouge": 0.25497224270827884 }, "prompt_5": { "rouge1": 0.37201110883916116, "rouge2": 0.12415105452309166, "rougeL": 0.29046671243562117, "avg_rouge": 0.262209625265958 } }, "dialogsum": { "prompt_1": { "rouge1": 0.24907880798174054, "rouge2": 0.04460441985657421, "rougeL": 0.199450203107373, "avg_rouge": 0.16437781031522924 }, "prompt_2": { "rouge1": 0.24714128996622692, "rouge2": 0.0434332879310033, "rougeL": 0.19917356862208666, "avg_rouge": 0.1632493821731056 }, "prompt_3": { "rouge1": 0.2486376312204472, "rouge2": 0.04400225966041297, "rougeL": 0.20042931696444927, "avg_rouge": 0.16435640261510315 }, "prompt_4": { "rouge1": 0.24435282003325978, "rouge2": 0.04255758734157635, "rougeL": 0.19604228754586672, "avg_rouge": 0.1609842316402343 }, "prompt_5": { "rouge1": 0.24330497216085947, "rouge2": 0.04101968290978953, "rougeL": 0.19525287677223835, "avg_rouge": 0.15985917728096247 } }, "sst2": { "prompt_1": { "accuracy": 0.9254587155963303 }, "prompt_2": { "accuracy": 0.944954128440367 }, "prompt_3": { "accuracy": 0.9311926605504587 }, "prompt_4": { "accuracy": 0.9243119266055045 }, "prompt_5": { "accuracy": 0.9013761467889908 } }, "cola": { "prompt_1": { "accuracy": 0.8581016299137104 }, "prompt_2": { "accuracy": 0.8600191754554171 }, "prompt_3": { "accuracy": 0.8523489932885906 }, "prompt_4": { "accuracy": 0.8552253116011506 }, "prompt_5": { "accuracy": 0.8581016299137104 } }, "qqp": { "prompt_1": { "accuracy": 0.7825 }, "prompt_2": { "accuracy": 0.7645 }, "prompt_3": { "accuracy": 0.789 }, "prompt_4": { "accuracy": 0.7835 }, "prompt_5": { "accuracy": 0.7625 } }, "mnli": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": -1, "prompt_4": -1, "prompt_5": { "accuracy": 0.3 } }, "qnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.6 } }, "wnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.7 } }, "mrpc": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "llama_2_7b_chat": { "model_size": "7B", "model_link": "https://huggingface.co/meta-llama/Llama-2-7b-chat-hf", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.7928571428571428, "language_acc": { "Spanish": 0.8109243697478992, "English": 0.8445378151260504, "Chinese": 0.7655462184873949, "Vietnamese": 0.7504201680672269 }, "consistency_score_2": 0.8026610644257702, "consistency_score_3": 0.7180672268907564, "consistency_score_4": 0.6672268907563025, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8647058823529412, "Spanish,Chinese": 0.7941176470588235, "Spanish,Vietnamese": 0.7848739495798319, "English,Chinese": 0.7924369747899159, "English,Vietnamese": 0.795798319327731, "Chinese,Vietnamese": 0.7840336134453781 }, "3_combine": { "Spanish,English,Chinese": 0.7378151260504202, "Spanish,English,Vietnamese": 0.7327731092436974, "Spanish,Chinese,Vietnamese": 0.6991596638655462, "English,Chinese,Vietnamese": 0.7025210084033613 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.6672268907563025 } }, "AC3_2": 0.7977289826947851, "AC3_3": 0.7536111552019897, "AC3_4": 0.7246372044724668 }, "prompt_2": { "overall_acc": 0.7834033613445378, "language_acc": { "Spanish": 0.7924369747899159, "English": 0.8445378151260504, "Chinese": 0.7747899159663866, "Vietnamese": 0.7218487394957983 }, "consistency_score_2": 0.7788515406162464, "consistency_score_3": 0.6844537815126049, "consistency_score_4": 0.626890756302521, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8428571428571429, "Spanish,Chinese": 0.7848739495798319, "Spanish,Vietnamese": 0.7504201680672269, "English,Chinese": 0.7857142857142857, "English,Vietnamese": 0.7554621848739496, "Chinese,Vietnamese": 0.753781512605042 }, "3_combine": { "Spanish,English,Chinese": 0.7201680672268908, "Spanish,English,Vietnamese": 0.6890756302521008, "Spanish,Chinese,Vietnamese": 0.6630252100840336, "English,Chinese,Vietnamese": 0.6655462184873949 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.626890756302521 } }, "AC3_2": 0.7811208197871916, "AC3_3": 0.7305934309679756, "AC3_4": 0.6964622762062693 }, "prompt_3": { "overall_acc": 0.7550420168067227, "language_acc": { "Spanish": 0.7596638655462185, "English": 0.8159663865546218, "Chinese": 0.7319327731092437, "Vietnamese": 0.7126050420168067 }, "consistency_score_2": 0.7333333333333334, "consistency_score_3": 0.6210084033613446, "consistency_score_4": 0.5487394957983194, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.8050420168067227, "Spanish,Chinese": 0.7159663865546219, "Spanish,Vietnamese": 0.7126050420168067, "English,Chinese": 0.7453781512605042, "English,Vietnamese": 0.7176470588235294, "Chinese,Vietnamese": 0.7033613445378152 }, "3_combine": { "Spanish,English,Chinese": 0.6512605042016807, "Spanish,English,Vietnamese": 0.6352941176470588, "Spanish,Chinese,Vietnamese": 0.5899159663865546, "English,Chinese,Vietnamese": 0.607563025210084 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5487394957983194 } }, "AC3_2": 0.7440293591294593, "AC3_3": 0.6814974661125117, "AC3_4": 0.6355687230898789 }, "prompt_4": { "overall_acc": 0.7894957983193277, "language_acc": { "Spanish": 0.8151260504201681, "English": 0.846218487394958, "Chinese": 0.7647058823529411, "Vietnamese": 0.7319327731092437 }, "consistency_score_2": 0.7942577030812324, "consistency_score_3": 0.706512605042017, "consistency_score_4": 0.6495798319327731, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.853781512605042, "Spanish,Chinese": 0.807563025210084, "Spanish,Vietnamese": 0.7756302521008404, "English,Chinese": 0.7949579831932773, "English,Vietnamese": 0.7714285714285715, "Chinese,Vietnamese": 0.7621848739495798 }, "3_combine": { "Spanish,English,Chinese": 0.7420168067226891, "Spanish,English,Vietnamese": 0.711764705882353, "Spanish,Chinese,Vietnamese": 0.692436974789916, "English,Chinese,Vietnamese": 0.6798319327731093 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.6495798319327731 } }, "AC3_2": 0.7918695917909816, "AC3_3": 0.7457026736611213, "AC3_4": 0.7127360607981694 }, "prompt_5": { "overall_acc": 0.7657563025210085, "language_acc": { "Spanish": 0.7932773109243697, "English": 0.8294117647058824, "Chinese": 0.7285714285714285, "Vietnamese": 0.711764705882353 }, "consistency_score_2": 0.7530812324929972, "consistency_score_3": 0.648109243697479, "consistency_score_4": 0.5831932773109244, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.83109243697479, "Spanish,Chinese": 0.7512605042016807, "Spanish,Vietnamese": 0.7327731092436974, "English,Chinese": 0.7504201680672269, "English,Vietnamese": 0.7378151260504202, "Chinese,Vietnamese": 0.7151260504201681 }, "3_combine": { "Spanish,English,Chinese": 0.6831932773109244, "Spanish,English,Vietnamese": 0.6672268907563025, "Spanish,Chinese,Vietnamese": 0.6201680672268908, "English,Chinese,Vietnamese": 0.6218487394957983 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5831932773109244 } }, "AC3_2": 0.7593658791801133, "AC3_3": 0.7020380960208695, "AC3_4": 0.6621210078307849 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.3961904761904762, "language_acc": { "Filipino": 0.38666666666666666, "Vietnamese": 0.3466666666666667, "Chinese": 0.34, "Spanish": 0.46, "Malay": 0.3933333333333333, "Indonesian": 0.38, "English": 0.4666666666666667 }, "consistency_score_2": 0.5098412698412699, "consistency_score_3": 0.32495238095238094, "consistency_score_4": 0.23085714285714282, "consistency_score_5": 0.1761904761904762, "consistency_score_6": 0.14190476190476192, "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.44666666666666666, "Filipino,Chinese": 0.43333333333333335, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.5133333333333333, "Filipino,Indonesian": 0.5, "Filipino,English": 0.54, "Vietnamese,Chinese": 0.5533333333333333, "Vietnamese,Spanish": 0.5333333333333333, "Vietnamese,Malay": 0.5666666666666667, "Vietnamese,Indonesian": 0.5866666666666667, "Vietnamese,English": 0.4866666666666667, "Chinese,Spanish": 0.5466666666666666, "Chinese,Malay": 0.47333333333333333, "Chinese,Indonesian": 0.52, "Chinese,English": 0.46, "Spanish,Malay": 0.44, "Spanish,Indonesian": 0.56, "Spanish,English": 0.5, "Malay,Indonesian": 0.6466666666666666, "Malay,English": 0.4066666666666667, "Indonesian,English": 0.5133333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.3, "Filipino,Vietnamese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Malay": 0.32666666666666666, "Filipino,Vietnamese,Indonesian": 0.3333333333333333, "Filipino,Vietnamese,English": 0.3, "Filipino,Chinese,Spanish": 0.29333333333333333, "Filipino,Chinese,Malay": 0.28, "Filipino,Chinese,Indonesian": 0.28, "Filipino,Chinese,English": 0.29333333333333333, "Filipino,Spanish,Malay": 0.26666666666666666, "Filipino,Spanish,Indonesian": 0.32666666666666666, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.37333333333333335, "Filipino,Malay,English": 0.2866666666666667, "Filipino,Indonesian,English": 0.34, "Vietnamese,Chinese,Spanish": 0.38666666666666666, "Vietnamese,Chinese,Malay": 0.36, "Vietnamese,Chinese,Indonesian": 0.38, "Vietnamese,Chinese,English": 0.30666666666666664, "Vietnamese,Spanish,Malay": 0.3333333333333333, "Vietnamese,Spanish,Indonesian": 0.4066666666666667, "Vietnamese,Spanish,English": 0.3333333333333333, "Vietnamese,Malay,Indonesian": 0.44666666666666666, "Vietnamese,Malay,English": 0.3, "Vietnamese,Indonesian,English": 0.3466666666666667, "Chinese,Spanish,Malay": 0.3, "Chinese,Spanish,Indonesian": 0.37333333333333335, "Chinese,Spanish,English": 0.30666666666666664, "Chinese,Malay,Indonesian": 0.36, "Chinese,Malay,English": 0.24666666666666667, "Chinese,Indonesian,English": 0.3, "Spanish,Malay,Indonesian": 0.36666666666666664, "Spanish,Malay,English": 0.24, "Spanish,Indonesian,English": 0.34, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.22, "Filipino,Vietnamese,Chinese,Malay": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian": 0.28, "Filipino,Vietnamese,Malay,English": 0.22, "Filipino,Vietnamese,Indonesian,English": 0.24, "Filipino,Chinese,Spanish,Malay": 0.19333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Malay,English": 0.19333333333333333, "Filipino,Chinese,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Spanish,Malay,English": 0.18666666666666668, "Filipino,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay": 0.26, "Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,Spanish,English": 0.22666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Chinese,Malay,English": 0.21333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, "Vietnamese,Spanish,Malay,Indonesian": 0.3, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian,English": 0.25333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, "Chinese,Spanish,Malay,English": 0.16, "Chinese,Spanish,Indonesian,English": 0.22, "Chinese,Malay,Indonesian,English": 0.2, "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,Spanish,English": 0.16, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.2, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, "AC3_2": 0.4458878098750368, "AC3_3": 0.3570528009969498, "AC3_4": 0.2917271306599506, "AC3_5": 0.24391094203804584, "AC3_6": 0.2089641803235764, "AC3_7": 0.18420664203073486 }, "prompt_2": { "overall_acc": 0.37238095238095237, "language_acc": { "Filipino": 0.38, "Vietnamese": 0.30666666666666664, "Chinese": 0.36666666666666664, "Spanish": 0.41333333333333333, "Malay": 0.32, "Indonesian": 0.36, "English": 0.46 }, "consistency_score_2": 0.4631746031746032, "consistency_score_3": 0.2681904761904762, "consistency_score_4": 0.17333333333333334, "consistency_score_5": 0.1184126984126984, "consistency_score_6": 0.08380952380952382, "consistency_score_7": 0.06, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.46, "Filipino,Chinese": 0.38666666666666666, "Filipino,Spanish": 0.5266666666666666, "Filipino,Malay": 0.44, "Filipino,Indonesian": 0.49333333333333335, "Filipino,English": 0.4533333333333333, "Vietnamese,Chinese": 0.4533333333333333, "Vietnamese,Spanish": 0.5, "Vietnamese,Malay": 0.46, "Vietnamese,Indonesian": 0.4533333333333333, "Vietnamese,English": 0.4666666666666667, "Chinese,Spanish": 0.48, "Chinese,Malay": 0.42, "Chinese,Indonesian": 0.44, "Chinese,English": 0.43333333333333335, "Spanish,Malay": 0.4666666666666667, "Spanish,Indonesian": 0.4866666666666667, "Spanish,English": 0.5066666666666667, "Malay,Indonesian": 0.5333333333333333, "Malay,English": 0.41333333333333333, "Indonesian,English": 0.4533333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.24666666666666667, "Filipino,Vietnamese,Spanish": 0.31333333333333335, "Filipino,Vietnamese,Malay": 0.26, "Filipino,Vietnamese,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,English": 0.26666666666666666, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.21333333333333335, "Filipino,Chinese,Indonesian": 0.23333333333333334, "Filipino,Chinese,English": 0.23333333333333334, "Filipino,Spanish,Malay": 0.26666666666666666, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.30666666666666664, "Filipino,Malay,Indonesian": 0.2866666666666667, "Filipino,Malay,English": 0.24, "Filipino,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish": 0.29333333333333333, "Vietnamese,Chinese,Malay": 0.24, "Vietnamese,Chinese,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,English": 0.25333333333333335, "Vietnamese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Spanish,Indonesian": 0.2733333333333333, "Vietnamese,Spanish,English": 0.30666666666666664, "Vietnamese,Malay,Indonesian": 0.29333333333333333, "Vietnamese,Malay,English": 0.24666666666666667, "Vietnamese,Indonesian,English": 0.25333333333333335, "Chinese,Spanish,Malay": 0.2733333333333333, "Chinese,Spanish,Indonesian": 0.26666666666666666, "Chinese,Spanish,English": 0.2733333333333333, "Chinese,Malay,Indonesian": 0.26666666666666666, "Chinese,Malay,English": 0.22666666666666666, "Chinese,Indonesian,English": 0.24666666666666667, "Spanish,Malay,Indonesian": 0.29333333333333333, "Spanish,Malay,English": 0.28, "Spanish,Indonesian,English": 0.2866666666666667, "Malay,Indonesian,English": 0.2733333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.14, "Filipino,Vietnamese,Chinese,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.18666666666666668, "Filipino,Chinese,Spanish,Indonesian": 0.18, "Filipino,Chinese,Spanish,English": 0.18, "Filipino,Chinese,Malay,Indonesian": 0.14, "Filipino,Chinese,Malay,English": 0.14666666666666667, "Filipino,Chinese,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Spanish,Malay,English": 0.18666666666666668, "Filipino,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay": 0.18, "Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, "Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, "Vietnamese,Spanish,Malay,Indonesian": 0.18, "Vietnamese,Spanish,Malay,English": 0.19333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Malay,Indonesian,English": 0.16666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Chinese,Spanish,Malay,English": 0.16666666666666666, "Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Chinese,Malay,Indonesian,English": 0.16, "Spanish,Malay,Indonesian,English": 0.18666666666666668 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.08, "Filipino,Vietnamese,Chinese,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Malay,English": 0.14, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12, "Filipino,Vietnamese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Chinese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.14, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.1, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 } }, "AC3_2": 0.4128448400142754, "AC3_3": 0.3118122990606781, "AC3_4": 0.23655613724577357, "AC3_5": 0.17968705719248965, "AC3_6": 0.13682473403900006, "AC3_7": 0.10334801759724328 }, "prompt_3": { "overall_acc": 0.3628571428571429, "language_acc": { "Filipino": 0.34, "Vietnamese": 0.3333333333333333, "Chinese": 0.3466666666666667, "Spanish": 0.38, "Malay": 0.30666666666666664, "Indonesian": 0.3466666666666667, "English": 0.4866666666666667 }, "consistency_score_2": 0.4231746031746032, "consistency_score_3": 0.22571428571428567, "consistency_score_4": 0.14057142857142857, "consistency_score_5": 0.09555555555555555, "consistency_score_6": 0.06761904761904762, "consistency_score_7": 0.04666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4266666666666667, "Filipino,Chinese": 0.4266666666666667, "Filipino,Spanish": 0.3466666666666667, "Filipino,Malay": 0.42, "Filipino,Indonesian": 0.46, "Filipino,English": 0.42, "Vietnamese,Chinese": 0.41333333333333333, "Vietnamese,Spanish": 0.3933333333333333, "Vietnamese,Malay": 0.36, "Vietnamese,Indonesian": 0.46, "Vietnamese,English": 0.44, "Chinese,Spanish": 0.4533333333333333, "Chinese,Malay": 0.34, "Chinese,Indonesian": 0.4533333333333333, "Chinese,English": 0.42, "Spanish,Malay": 0.38, "Spanish,Indonesian": 0.46, "Spanish,English": 0.5, "Malay,Indonesian": 0.47333333333333333, "Malay,English": 0.38666666666666666, "Indonesian,English": 0.4533333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.24, "Filipino,Vietnamese,Spanish": 0.2, "Filipino,Vietnamese,Malay": 0.18, "Filipino,Vietnamese,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,English": 0.23333333333333334, "Filipino,Chinese,Spanish": 0.20666666666666667, "Filipino,Chinese,Malay": 0.18666666666666668, "Filipino,Chinese,Indonesian": 0.26666666666666666, "Filipino,Chinese,English": 0.22666666666666666, "Filipino,Spanish,Malay": 0.17333333333333334, "Filipino,Spanish,Indonesian": 0.22666666666666666, "Filipino,Spanish,English": 0.22, "Filipino,Malay,Indonesian": 0.24, "Filipino,Malay,English": 0.22, "Filipino,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish": 0.24666666666666667, "Vietnamese,Chinese,Malay": 0.17333333333333334, "Vietnamese,Chinese,Indonesian": 0.28, "Vietnamese,Chinese,English": 0.26, "Vietnamese,Spanish,Malay": 0.17333333333333334, "Vietnamese,Spanish,Indonesian": 0.25333333333333335, "Vietnamese,Spanish,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Malay,English": 0.17333333333333334, "Vietnamese,Indonesian,English": 0.26, "Chinese,Spanish,Malay": 0.18, "Chinese,Spanish,Indonesian": 0.26, "Chinese,Spanish,English": 0.25333333333333335, "Chinese,Malay,Indonesian": 0.22, "Chinese,Malay,English": 0.16, "Chinese,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian": 0.22, "Spanish,Malay,English": 0.22, "Spanish,Indonesian,English": 0.2733333333333333, "Malay,Indonesian,English": 0.24666666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.14, "Filipino,Vietnamese,Chinese,Malay": 0.12, "Filipino,Vietnamese,Chinese,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,English": 0.18, "Filipino,Vietnamese,Spanish,Malay": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.1, "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Chinese,Spanish,English": 0.14, "Filipino,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Chinese,Malay,English": 0.12, "Filipino,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian": 0.11333333333333333, "Filipino,Spanish,Malay,English": 0.13333333333333333, "Filipino,Spanish,Indonesian,English": 0.16, "Filipino,Malay,Indonesian,English": 0.16, "Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,Spanish,English": 0.17333333333333334, "Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Vietnamese,Chinese,Malay,English": 0.11333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian": 0.11333333333333333, "Vietnamese,Spanish,Malay,English": 0.11333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Malay,Indonesian,English": 0.14, "Chinese,Spanish,Malay,Indonesian": 0.12, "Chinese,Spanish,Malay,English": 0.10666666666666667, "Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Chinese,Malay,Indonesian,English": 0.11333333333333333, "Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.06666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.08, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.08, "Filipino,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, "Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Vietnamese,Spanish,Malay,Indonesian,English": 0.08666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.05333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.06, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.06666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04666666666666667 } }, "AC3_2": 0.39070159238048136, "AC3_3": 0.2783079056392611, "AC3_4": 0.20263985726475858, "AC3_5": 0.1512742381941472, "AC3_6": 0.11399494308350622, "AC3_7": 0.08269767439841103 }, "prompt_4": { "overall_acc": 0.4066666666666667, "language_acc": { "Filipino": 0.37333333333333335, "Vietnamese": 0.36666666666666664, "Chinese": 0.32666666666666666, "Spanish": 0.4666666666666667, "Malay": 0.3933333333333333, "Indonesian": 0.37333333333333335, "English": 0.5466666666666666 }, "consistency_score_2": 0.47174603174603175, "consistency_score_3": 0.2876190476190476, "consistency_score_4": 0.19466666666666663, "consistency_score_5": 0.13809523809523808, "consistency_score_6": 0.1, "consistency_score_7": 0.07333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4666666666666667, "Filipino,Chinese": 0.4266666666666667, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.43333333333333335, "Filipino,Indonesian": 0.4266666666666667, "Filipino,English": 0.5, "Vietnamese,Chinese": 0.5, "Vietnamese,Spanish": 0.47333333333333333, "Vietnamese,Malay": 0.4866666666666667, "Vietnamese,Indonesian": 0.5, "Vietnamese,English": 0.47333333333333333, "Chinese,Spanish": 0.4533333333333333, "Chinese,Malay": 0.44, "Chinese,Indonesian": 0.4533333333333333, "Chinese,English": 0.47333333333333333, "Spanish,Malay": 0.4666666666666667, "Spanish,Indonesian": 0.4666666666666667, "Spanish,English": 0.54, "Malay,Indonesian": 0.5333333333333333, "Malay,English": 0.47333333333333333, "Indonesian,English": 0.4666666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.28, "Filipino,Vietnamese,Spanish": 0.28, "Filipino,Vietnamese,Malay": 0.2733333333333333, "Filipino,Vietnamese,Indonesian": 0.26, "Filipino,Vietnamese,English": 0.30666666666666664, "Filipino,Chinese,Spanish": 0.2733333333333333, "Filipino,Chinese,Malay": 0.24666666666666667, "Filipino,Chinese,Indonesian": 0.23333333333333334, "Filipino,Chinese,English": 0.28, "Filipino,Spanish,Malay": 0.26, "Filipino,Spanish,Indonesian": 0.26, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.2733333333333333, "Filipino,Malay,English": 0.29333333333333333, "Filipino,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish": 0.2866666666666667, "Vietnamese,Chinese,Malay": 0.29333333333333333, "Vietnamese,Chinese,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,English": 0.29333333333333333, "Vietnamese,Spanish,Malay": 0.3, "Vietnamese,Spanish,Indonesian": 0.3, "Vietnamese,Spanish,English": 0.31333333333333335, "Vietnamese,Malay,Indonesian": 0.3333333333333333, "Vietnamese,Malay,English": 0.30666666666666664, "Vietnamese,Indonesian,English": 0.28, "Chinese,Spanish,Malay": 0.26666666666666666, "Chinese,Spanish,Indonesian": 0.28, "Chinese,Spanish,English": 0.30666666666666664, "Chinese,Malay,Indonesian": 0.3, "Chinese,Malay,English": 0.26666666666666666, "Chinese,Indonesian,English": 0.26, "Spanish,Malay,Indonesian": 0.32666666666666666, "Spanish,Malay,English": 0.3, "Spanish,Indonesian,English": 0.31333333333333335, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.2, "Filipino,Vietnamese,Chinese,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,English": 0.2, "Filipino,Vietnamese,Spanish,Malay": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.16666666666666666, "Filipino,Chinese,Malay,English": 0.18666666666666668, "Filipino,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Spanish,Malay,English": 0.21333333333333335, "Filipino,Spanish,Indonesian,English": 0.2, "Filipino,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Indonesian": 0.2, "Vietnamese,Chinese,Spanish,English": 0.21333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,Malay,English": 0.2, "Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, "Vietnamese,Spanish,Malay,Indonesian": 0.22, "Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Vietnamese,Malay,Indonesian,English": 0.22, "Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Chinese,Spanish,Malay,English": 0.18666666666666668, "Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Chinese,Malay,Indonesian,English": 0.18666666666666668, "Spanish,Malay,Indonesian,English": 0.22666666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 } }, "AC3_2": 0.4367955667494553, "AC3_3": 0.336936442566925, "AC3_4": 0.26329637837454417, "AC3_5": 0.20617715613930893, "AC3_6": 0.16052631575779086, "AC3_7": 0.12425925923337192 }, "prompt_5": { "overall_acc": 0.39904761904761904, "language_acc": { "Filipino": 0.38, "Vietnamese": 0.36666666666666664, "Chinese": 0.3466666666666667, "Spanish": 0.4266666666666667, "Malay": 0.41333333333333333, "Indonesian": 0.37333333333333335, "English": 0.4866666666666667 }, "consistency_score_2": 0.4479365079365079, "consistency_score_3": 0.2681904761904762, "consistency_score_4": 0.1824761904761905, "consistency_score_5": 0.13174603174603175, "consistency_score_6": 0.0980952380952381, "consistency_score_7": 0.07333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4266666666666667, "Filipino,Chinese": 0.4266666666666667, "Filipino,Spanish": 0.42, "Filipino,Malay": 0.44, "Filipino,Indonesian": 0.44666666666666666, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.4266666666666667, "Vietnamese,Spanish": 0.4066666666666667, "Vietnamese,Malay": 0.44666666666666666, "Vietnamese,Indonesian": 0.5, "Vietnamese,English": 0.49333333333333335, "Chinese,Spanish": 0.4266666666666667, "Chinese,Malay": 0.41333333333333333, "Chinese,Indonesian": 0.41333333333333333, "Chinese,English": 0.41333333333333333, "Spanish,Malay": 0.44, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.5333333333333333, "Malay,Indonesian": 0.5, "Malay,English": 0.4266666666666667, "Indonesian,English": 0.44666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.26, "Filipino,Vietnamese,Spanish": 0.24, "Filipino,Vietnamese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Indonesian": 0.26, "Filipino,Vietnamese,English": 0.2733333333333333, "Filipino,Chinese,Spanish": 0.23333333333333334, "Filipino,Chinese,Malay": 0.24666666666666667, "Filipino,Chinese,Indonesian": 0.22666666666666666, "Filipino,Chinese,English": 0.28, "Filipino,Spanish,Malay": 0.25333333333333335, "Filipino,Spanish,Indonesian": 0.2733333333333333, "Filipino,Spanish,English": 0.30666666666666664, "Filipino,Malay,Indonesian": 0.25333333333333335, "Filipino,Malay,English": 0.2733333333333333, "Filipino,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish": 0.24, "Vietnamese,Chinese,Malay": 0.26, "Vietnamese,Chinese,Indonesian": 0.26, "Vietnamese,Chinese,English": 0.26666666666666666, "Vietnamese,Spanish,Malay": 0.26, "Vietnamese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,English": 0.30666666666666664, "Vietnamese,Malay,Indonesian": 0.31333333333333335, "Vietnamese,Malay,English": 0.29333333333333333, "Vietnamese,Indonesian,English": 0.3, "Chinese,Spanish,Malay": 0.24, "Chinese,Spanish,Indonesian": 0.26666666666666666, "Chinese,Spanish,English": 0.26666666666666666, "Chinese,Malay,Indonesian": 0.2733333333333333, "Chinese,Malay,English": 0.24666666666666667, "Chinese,Indonesian,English": 0.24666666666666667, "Spanish,Malay,Indonesian": 0.29333333333333333, "Spanish,Malay,English": 0.28, "Spanish,Indonesian,English": 0.30666666666666664, "Malay,Indonesian,English": 0.2733333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.18, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Malay,English": 0.2, "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay": 0.16666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.16, "Filipino,Chinese,Malay,English": 0.18666666666666668, "Filipino,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.16, "Filipino,Spanish,Malay,English": 0.19333333333333333, "Filipino,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Spanish,English": 0.18666666666666668, "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Malay,English": 0.19333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.22, "Vietnamese,Malay,Indonesian,English": 0.22, "Chinese,Spanish,Malay,Indonesian": 0.18, "Chinese,Spanish,Malay,English": 0.16666666666666666, "Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Chinese,Malay,Indonesian,English": 0.18, "Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,English": 0.14, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Chinese,Spanish,Malay,Indonesian,English": 0.12 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 } }, "AC3_2": 0.42208110225615647, "AC3_3": 0.3207873522893754, "AC3_4": 0.2504340772891452, "AC3_5": 0.1980918204229216, "AC3_6": 0.1574785622744718, "AC3_7": 0.12389784943613723 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3766233766233766, "language_acc": { "Indonesian": 0.3693181818181818, "English": 0.44886363636363635, "Filipino": 0.32386363636363635, "Spanish": 0.38636363636363635, "Chinese": 0.3806818181818182, "Malay": 0.35795454545454547, "Vietnamese": 0.3693181818181818 }, "consistency_score_2": 0.5459956709956709, "consistency_score_3": 0.3681818181818183, "consistency_score_4": 0.27094155844155837, "consistency_score_5": 0.2077922077922078, "consistency_score_6": 0.16152597402597402, "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5340909090909091, "Indonesian,Filipino": 0.5454545454545454, "Indonesian,Spanish": 0.6079545454545454, "Indonesian,Chinese": 0.5454545454545454, "Indonesian,Malay": 0.625, "Indonesian,Vietnamese": 0.6079545454545454, "English,Filipino": 0.44886363636363635, "English,Spanish": 0.6477272727272727, "English,Chinese": 0.5568181818181818, "English,Malay": 0.48863636363636365, "English,Vietnamese": 0.4715909090909091, "Filipino,Spanish": 0.5227272727272727, "Filipino,Chinese": 0.4602272727272727, "Filipino,Malay": 0.5625, "Filipino,Vietnamese": 0.4772727272727273, "Spanish,Chinese": 0.5454545454545454, "Spanish,Malay": 0.5852272727272727, "Spanish,Vietnamese": 0.5454545454545454, "Chinese,Malay": 0.5625, "Chinese,Vietnamese": 0.5909090909090909, "Malay,Vietnamese": 0.5340909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.32954545454545453, "Indonesian,English,Spanish": 0.4431818181818182, "Indonesian,English,Chinese": 0.38636363636363635, "Indonesian,English,Malay": 0.3806818181818182, "Indonesian,English,Vietnamese": 0.3693181818181818, "Indonesian,Filipino,Spanish": 0.38636363636363635, "Indonesian,Filipino,Chinese": 0.32954545454545453, "Indonesian,Filipino,Malay": 0.4090909090909091, "Indonesian,Filipino,Vietnamese": 0.35795454545454547, "Indonesian,Spanish,Chinese": 0.3977272727272727, "Indonesian,Spanish,Malay": 0.4375, "Indonesian,Spanish,Vietnamese": 0.4147727272727273, "Indonesian,Chinese,Malay": 0.4147727272727273, "Indonesian,Chinese,Vietnamese": 0.4034090909090909, "Indonesian,Malay,Vietnamese": 0.4147727272727273, "English,Filipino,Spanish": 0.3522727272727273, "English,Filipino,Chinese": 0.3068181818181818, "English,Filipino,Malay": 0.30113636363636365, "English,Filipino,Vietnamese": 0.25, "English,Spanish,Chinese": 0.42613636363636365, "English,Spanish,Malay": 0.3977272727272727, "English,Spanish,Vietnamese": 0.375, "English,Chinese,Malay": 0.375, "English,Chinese,Vietnamese": 0.36363636363636365, "English,Malay,Vietnamese": 0.3125, "Filipino,Spanish,Chinese": 0.32954545454545453, "Filipino,Spanish,Malay": 0.3693181818181818, "Filipino,Spanish,Vietnamese": 0.32386363636363635, "Filipino,Chinese,Malay": 0.3409090909090909, "Filipino,Chinese,Vietnamese": 0.30113636363636365, "Filipino,Malay,Vietnamese": 0.32386363636363635, "Spanish,Chinese,Malay": 0.4034090909090909, "Spanish,Chinese,Vietnamese": 0.3806818181818182, "Spanish,Malay,Vietnamese": 0.375, "Chinese,Malay,Vietnamese": 0.4034090909090909 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2840909090909091, "Indonesian,English,Filipino,Chinese": 0.25, "Indonesian,English,Filipino,Malay": 0.25, "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Chinese": 0.3352272727272727, "Indonesian,English,Spanish,Malay": 0.32954545454545453, "Indonesian,English,Spanish,Vietnamese": 0.3068181818181818, "Indonesian,English,Chinese,Malay": 0.3181818181818182, "Indonesian,English,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,English,Malay,Vietnamese": 0.2727272727272727, "Indonesian,Filipino,Spanish,Chinese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Malay": 0.30113636363636365, "Indonesian,Filipino,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,Filipino,Chinese,Malay": 0.2727272727272727, "Indonesian,Filipino,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Chinese,Malay": 0.32954545454545453, "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Malay,Vietnamese": 0.3068181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.32954545454545453, "English,Filipino,Spanish,Chinese": 0.26136363636363635, "English,Filipino,Spanish,Malay": 0.25, "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, "English,Filipino,Chinese,Malay": 0.22727272727272727, "English,Filipino,Chinese,Vietnamese": 0.19886363636363635, "English,Filipino,Malay,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese,Malay": 0.3181818181818182, "English,Spanish,Chinese,Vietnamese": 0.29545454545454547, "English,Spanish,Malay,Vietnamese": 0.25, "English,Chinese,Malay,Vietnamese": 0.2784090909090909, "Filipino,Spanish,Chinese,Malay": 0.25, "Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, "Spanish,Chinese,Malay,Vietnamese": 0.2897727272727273 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2215909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,English,Filipino,Chinese,Malay": 0.20454545454545456, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Spanish,Chinese,Malay": 0.2784090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.22727272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, "AC3_2": 0.44576303457102456, "AC3_3": 0.3723547594015401, "AC3_4": 0.31515889470410957, "AC3_5": 0.2678210677752407, "AC3_6": 0.22608763781033162, "AC3_7": 0.1877022653347493 }, "prompt_2": { "overall_acc": 0.37581168831168826, "language_acc": { "Indonesian": 0.3465909090909091, "English": 0.4715909090909091, "Filipino": 0.3409090909090909, "Spanish": 0.3806818181818182, "Chinese": 0.3977272727272727, "Malay": 0.32386363636363635, "Vietnamese": 0.3693181818181818 }, "consistency_score_2": 0.5165043290043292, "consistency_score_3": 0.3339285714285714, "consistency_score_4": 0.2387987012987013, "consistency_score_5": 0.1810064935064935, "consistency_score_6": 0.14204545454545456, "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5681818181818182, "Indonesian,Filipino": 0.5170454545454546, "Indonesian,Spanish": 0.5795454545454546, "Indonesian,Chinese": 0.5113636363636364, "Indonesian,Malay": 0.6079545454545454, "Indonesian,Vietnamese": 0.5284090909090909, "English,Filipino": 0.4431818181818182, "English,Spanish": 0.6022727272727273, "English,Chinese": 0.5397727272727273, "English,Malay": 0.48295454545454547, "English,Vietnamese": 0.4943181818181818, "Filipino,Spanish": 0.4659090909090909, "Filipino,Chinese": 0.4318181818181818, "Filipino,Malay": 0.5568181818181818, "Filipino,Vietnamese": 0.4772727272727273, "Spanish,Chinese": 0.5397727272727273, "Spanish,Malay": 0.5397727272727273, "Spanish,Vietnamese": 0.45454545454545453, "Chinese,Malay": 0.5113636363636364, "Chinese,Vietnamese": 0.4659090909090909, "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.3409090909090909, "Indonesian,English,Spanish": 0.4318181818181818, "Indonesian,English,Chinese": 0.3693181818181818, "Indonesian,English,Malay": 0.3806818181818182, "Indonesian,English,Vietnamese": 0.35795454545454547, "Indonesian,Filipino,Spanish": 0.3409090909090909, "Indonesian,Filipino,Chinese": 0.29545454545454547, "Indonesian,Filipino,Malay": 0.3806818181818182, "Indonesian,Filipino,Vietnamese": 0.3409090909090909, "Indonesian,Spanish,Chinese": 0.375, "Indonesian,Spanish,Malay": 0.3977272727272727, "Indonesian,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,Chinese,Malay": 0.375, "Indonesian,Chinese,Vietnamese": 0.3125, "Indonesian,Malay,Vietnamese": 0.375, "English,Filipino,Spanish": 0.3068181818181818, "English,Filipino,Chinese": 0.2840909090909091, "English,Filipino,Malay": 0.3068181818181818, "English,Filipino,Vietnamese": 0.2727272727272727, "English,Spanish,Chinese": 0.38636363636363635, "English,Spanish,Malay": 0.3693181818181818, "English,Spanish,Vietnamese": 0.32954545454545453, "English,Chinese,Malay": 0.32386363636363635, "English,Chinese,Vietnamese": 0.3181818181818182, "English,Malay,Vietnamese": 0.3125, "Filipino,Spanish,Chinese": 0.29545454545454547, "Filipino,Spanish,Malay": 0.32954545454545453, "Filipino,Spanish,Vietnamese": 0.2727272727272727, "Filipino,Chinese,Malay": 0.32386363636363635, "Filipino,Chinese,Vietnamese": 0.2556818181818182, "Filipino,Malay,Vietnamese": 0.32954545454545453, "Spanish,Chinese,Malay": 0.35795454545454547, "Spanish,Chinese,Vietnamese": 0.2897727272727273, "Spanish,Malay,Vietnamese": 0.3068181818181818, "Chinese,Malay,Vietnamese": 0.3125 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2727272727272727, "Indonesian,English,Filipino,Chinese": 0.23295454545454544, "Indonesian,English,Filipino,Malay": 0.26136363636363635, "Indonesian,English,Filipino,Vietnamese": 0.23863636363636365, "Indonesian,English,Spanish,Chinese": 0.3068181818181818, "Indonesian,English,Spanish,Malay": 0.30113636363636365, "Indonesian,English,Spanish,Vietnamese": 0.26136363636363635, "Indonesian,English,Chinese,Malay": 0.2784090909090909, "Indonesian,English,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese": 0.2159090909090909, "Indonesian,Filipino,Spanish,Malay": 0.25, "Indonesian,Filipino,Spanish,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Chinese,Malay": 0.23863636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Chinese,Malay": 0.2784090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.25, "Indonesian,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese": 0.22727272727272727, "English,Filipino,Spanish,Malay": 0.2215909090909091, "English,Filipino,Spanish,Vietnamese": 0.20454545454545456, "English,Filipino,Chinese,Malay": 0.21022727272727273, "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, "English,Filipino,Malay,Vietnamese": 0.2159090909090909, "English,Spanish,Chinese,Malay": 0.2727272727272727, "English,Spanish,Chinese,Vietnamese": 0.22727272727272727, "English,Spanish,Malay,Vietnamese": 0.23863636363636365, "English,Chinese,Malay,Vietnamese": 0.22727272727272727, "Filipino,Spanish,Chinese,Malay": 0.23863636363636365, "Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1875, "Indonesian,English,Filipino,Chinese,Malay": 0.1875, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Malay,Vietnamese": 0.20454545454545456, "Indonesian,English,Chinese,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, "AC3_2": 0.43506641170807325, "AC3_3": 0.3536343287960766, "AC3_4": 0.29203327702934345, "AC3_5": 0.24433238040366068, "AC3_6": 0.20616628665155365, "AC3_7": 0.17450625656015512 }, "prompt_3": { "overall_acc": 0.3701298701298702, "language_acc": { "Indonesian": 0.35795454545454547, "English": 0.4147727272727273, "Filipino": 0.3181818181818182, "Spanish": 0.4090909090909091, "Chinese": 0.36363636363636365, "Malay": 0.36363636363636365, "Vietnamese": 0.36363636363636365 }, "consistency_score_2": 0.4948593073593073, "consistency_score_3": 0.3042207792207793, "consistency_score_4": 0.20519480519480515, "consistency_score_5": 0.1442099567099567, "consistency_score_6": 0.1030844155844156, "consistency_score_7": 0.07386363636363637, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.48295454545454547, "Indonesian,Filipino": 0.4772727272727273, "Indonesian,Spanish": 0.5397727272727273, "Indonesian,Chinese": 0.5284090909090909, "Indonesian,Malay": 0.6363636363636364, "Indonesian,Vietnamese": 0.5056818181818182, "English,Filipino": 0.38636363636363635, "English,Spanish": 0.5795454545454546, "English,Chinese": 0.5568181818181818, "English,Malay": 0.45454545454545453, "English,Vietnamese": 0.3409090909090909, "Filipino,Spanish": 0.48863636363636365, "Filipino,Chinese": 0.4375, "Filipino,Malay": 0.5056818181818182, "Filipino,Vietnamese": 0.4772727272727273, "Spanish,Chinese": 0.5340909090909091, "Spanish,Malay": 0.5, "Spanish,Vietnamese": 0.4715909090909091, "Chinese,Malay": 0.5227272727272727, "Chinese,Vietnamese": 0.4715909090909091, "Malay,Vietnamese": 0.4943181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.2556818181818182, "Indonesian,English,Spanish": 0.3693181818181818, "Indonesian,English,Chinese": 0.32954545454545453, "Indonesian,English,Malay": 0.3181818181818182, "Indonesian,English,Vietnamese": 0.25, "Indonesian,Filipino,Spanish": 0.3125, "Indonesian,Filipino,Chinese": 0.2727272727272727, "Indonesian,Filipino,Malay": 0.36363636363636365, "Indonesian,Filipino,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Chinese": 0.3522727272727273, "Indonesian,Spanish,Malay": 0.3806818181818182, "Indonesian,Spanish,Vietnamese": 0.3068181818181818, "Indonesian,Chinese,Malay": 0.38636363636363635, "Indonesian,Chinese,Vietnamese": 0.3068181818181818, "Indonesian,Malay,Vietnamese": 0.3806818181818182, "English,Filipino,Spanish": 0.2897727272727273, "English,Filipino,Chinese": 0.25, "English,Filipino,Malay": 0.26136363636363635, "English,Filipino,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese": 0.35795454545454547, "English,Spanish,Malay": 0.32386363636363635, "English,Spanish,Vietnamese": 0.2727272727272727, "English,Chinese,Malay": 0.3125, "English,Chinese,Vietnamese": 0.2556818181818182, "English,Malay,Vietnamese": 0.2215909090909091, "Filipino,Spanish,Chinese": 0.2897727272727273, "Filipino,Spanish,Malay": 0.3125, "Filipino,Spanish,Vietnamese": 0.30113636363636365, "Filipino,Chinese,Malay": 0.3068181818181818, "Filipino,Chinese,Vietnamese": 0.2727272727272727, "Filipino,Malay,Vietnamese": 0.30113636363636365, "Spanish,Chinese,Malay": 0.3409090909090909, "Spanish,Chinese,Vietnamese": 0.30113636363636365, "Spanish,Malay,Vietnamese": 0.2897727272727273, "Chinese,Malay,Vietnamese": 0.3068181818181818 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.20454545454545456, "Indonesian,English,Filipino,Chinese": 0.17045454545454544, "Indonesian,English,Filipino,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese": 0.24431818181818182, "Indonesian,English,Spanish,Malay": 0.23863636363636365, "Indonesian,English,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,English,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Chinese,Vietnamese": 0.1875, "Indonesian,English,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay": 0.23295454545454544, "Indonesian,Filipino,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Spanish,Chinese,Malay": 0.2727272727272727, "Indonesian,Spanish,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.24431818181818182, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.20454545454545456, "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, "English,Filipino,Chinese,Malay": 0.1875, "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Malay,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese,Malay": 0.23295454545454544, "English,Spanish,Chinese,Vietnamese": 0.20454545454545456, "English,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Chinese,Malay,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Chinese,Malay": 0.2159090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.1875, "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, "English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 } }, "AC3_2": 0.4235017406251294, "AC3_3": 0.3339544422342411, "AC3_4": 0.26402040392936654, "AC3_5": 0.20755309914872694, "AC3_6": 0.16125726756967432, "AC3_7": 0.12315107193503502 }, "prompt_4": { "overall_acc": 0.3733766233766233, "language_acc": { "Indonesian": 0.32954545454545453, "English": 0.4090909090909091, "Filipino": 0.3522727272727273, "Spanish": 0.39204545454545453, "Chinese": 0.3977272727272727, "Malay": 0.3465909090909091, "Vietnamese": 0.38636363636363635 }, "consistency_score_2": 0.49621212121212127, "consistency_score_3": 0.3079545454545454, "consistency_score_4": 0.21266233766233764, "consistency_score_5": 0.15611471861471865, "consistency_score_6": 0.11850649350649352, "consistency_score_7": 0.09090909090909091, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4772727272727273, "Indonesian,Filipino": 0.45454545454545453, "Indonesian,Spanish": 0.5227272727272727, "Indonesian,Chinese": 0.5284090909090909, "Indonesian,Malay": 0.5681818181818182, "Indonesian,Vietnamese": 0.5511363636363636, "English,Filipino": 0.4034090909090909, "English,Spanish": 0.5340909090909091, "English,Chinese": 0.4659090909090909, "English,Malay": 0.4318181818181818, "English,Vietnamese": 0.48295454545454547, "Filipino,Spanish": 0.4772727272727273, "Filipino,Chinese": 0.4715909090909091, "Filipino,Malay": 0.5170454545454546, "Filipino,Vietnamese": 0.4659090909090909, "Spanish,Chinese": 0.5340909090909091, "Spanish,Malay": 0.5170454545454546, "Spanish,Vietnamese": 0.4715909090909091, "Chinese,Malay": 0.5, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { "Indonesian,English,Filipino": 0.25, "Indonesian,English,Spanish": 0.3352272727272727, "Indonesian,English,Chinese": 0.3181818181818182, "Indonesian,English,Malay": 0.3125, "Indonesian,English,Vietnamese": 0.3352272727272727, "Indonesian,Filipino,Spanish": 0.29545454545454547, "Indonesian,Filipino,Chinese": 0.30113636363636365, "Indonesian,Filipino,Malay": 0.3181818181818182, "Indonesian,Filipino,Vietnamese": 0.29545454545454547, "Indonesian,Spanish,Chinese": 0.3409090909090909, "Indonesian,Spanish,Malay": 0.35795454545454547, "Indonesian,Spanish,Vietnamese": 0.32386363636363635, "Indonesian,Chinese,Malay": 0.35795454545454547, "Indonesian,Chinese,Vietnamese": 0.3522727272727273, "Indonesian,Malay,Vietnamese": 0.3693181818181818, "English,Filipino,Spanish": 0.26704545454545453, "English,Filipino,Chinese": 0.26704545454545453, "English,Filipino,Malay": 0.25, "English,Filipino,Vietnamese": 0.25, "English,Spanish,Chinese": 0.3352272727272727, "English,Spanish,Malay": 0.2840909090909091, "English,Spanish,Vietnamese": 0.30113636363636365, "English,Chinese,Malay": 0.2784090909090909, "English,Chinese,Vietnamese": 0.3068181818181818, "English,Malay,Vietnamese": 0.29545454545454547, "Filipino,Spanish,Chinese": 0.3181818181818182, "Filipino,Spanish,Malay": 0.32386363636363635, "Filipino,Spanish,Vietnamese": 0.26704545454545453, "Filipino,Chinese,Malay": 0.3125, "Filipino,Chinese,Vietnamese": 0.29545454545454547, "Filipino,Malay,Vietnamese": 0.29545454545454547, "Spanish,Chinese,Malay": 0.32954545454545453, "Spanish,Chinese,Vietnamese": 0.3181818181818182, "Spanish,Malay,Vietnamese": 0.30113636363636365, "Chinese,Malay,Vietnamese": 0.3181818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1875, "Indonesian,English,Filipino,Chinese": 0.19318181818181818, "Indonesian,English,Filipino,Malay": 0.1875, "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese": 0.25, "Indonesian,English,Spanish,Malay": 0.22727272727272727, "Indonesian,English,Spanish,Vietnamese": 0.2215909090909091, "Indonesian,English,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,English,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Filipino,Spanish,Chinese": 0.2159090909090909, "Indonesian,Filipino,Spanish,Malay": 0.23295454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Chinese,Malay": 0.23295454545454544, "Indonesian,Filipino,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Chinese,Malay,Vietnamese": 0.26136363636363635, "English,Filipino,Spanish,Chinese": 0.20454545454545456, "English,Filipino,Spanish,Malay": 0.18181818181818182, "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay": 0.17045454545454544, "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, "English,Filipino,Malay,Vietnamese": 0.17613636363636365, "English,Spanish,Chinese,Malay": 0.2159090909090909, "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, "English,Spanish,Malay,Vietnamese": 0.19318181818181818, "English,Chinese,Malay,Vietnamese": 0.21022727272727273, "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.1875, "Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, "Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese,Malay": 0.1875, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.125, "English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 } }, "AC3_2": 0.42611868524827656, "AC3_3": 0.33752463876173305, "AC3_4": 0.27098248007751463, "AC3_5": 0.22017200985960544, "AC3_6": 0.179910848191104, "AC3_7": 0.1462174189131987 }, "prompt_5": { "overall_acc": 0.35227272727272724, "language_acc": { "Indonesian": 0.32954545454545453, "English": 0.44886363636363635, "Filipino": 0.3068181818181818, "Spanish": 0.3409090909090909, "Chinese": 0.3465909090909091, "Malay": 0.32954545454545453, "Vietnamese": 0.36363636363636365 }, "consistency_score_2": 0.4710497835497835, "consistency_score_3": 0.28035714285714286, "consistency_score_4": 0.1850649350649351, "consistency_score_5": 0.1304112554112554, "consistency_score_6": 0.09659090909090909, "consistency_score_7": 0.07386363636363637, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.48295454545454547, "Indonesian,Filipino": 0.4659090909090909, "Indonesian,Spanish": 0.5397727272727273, "Indonesian,Chinese": 0.48863636363636365, "Indonesian,Malay": 0.5454545454545454, "Indonesian,Vietnamese": 0.5170454545454546, "English,Filipino": 0.42613636363636365, "English,Spanish": 0.5284090909090909, "English,Chinese": 0.4772727272727273, "English,Malay": 0.4090909090909091, "English,Vietnamese": 0.4602272727272727, "Filipino,Spanish": 0.4090909090909091, "Filipino,Chinese": 0.4431818181818182, "Filipino,Malay": 0.45454545454545453, "Filipino,Vietnamese": 0.4375, "Spanish,Chinese": 0.5, "Spanish,Malay": 0.45454545454545453, "Spanish,Vietnamese": 0.48863636363636365, "Chinese,Malay": 0.4659090909090909, "Chinese,Vietnamese": 0.42613636363636365, "Malay,Vietnamese": 0.4715909090909091 }, "3_combine": { "Indonesian,English,Filipino": 0.26136363636363635, "Indonesian,English,Spanish": 0.3465909090909091, "Indonesian,English,Chinese": 0.3125, "Indonesian,English,Malay": 0.2897727272727273, "Indonesian,English,Vietnamese": 0.3181818181818182, "Indonesian,Filipino,Spanish": 0.2784090909090909, "Indonesian,Filipino,Chinese": 0.26704545454545453, "Indonesian,Filipino,Malay": 0.2840909090909091, "Indonesian,Filipino,Vietnamese": 0.2897727272727273, "Indonesian,Spanish,Chinese": 0.3352272727272727, "Indonesian,Spanish,Malay": 0.32386363636363635, "Indonesian,Spanish,Vietnamese": 0.3352272727272727, "Indonesian,Chinese,Malay": 0.3181818181818182, "Indonesian,Chinese,Vietnamese": 0.2784090909090909, "Indonesian,Malay,Vietnamese": 0.3125, "English,Filipino,Spanish": 0.26136363636363635, "English,Filipino,Chinese": 0.2784090909090909, "English,Filipino,Malay": 0.24431818181818182, "English,Filipino,Vietnamese": 0.24431818181818182, "English,Spanish,Chinese": 0.3181818181818182, "English,Spanish,Malay": 0.26704545454545453, "English,Spanish,Vietnamese": 0.32386363636363635, "English,Chinese,Malay": 0.2556818181818182, "English,Chinese,Vietnamese": 0.26136363636363635, "English,Malay,Vietnamese": 0.25, "Filipino,Spanish,Chinese": 0.2556818181818182, "Filipino,Spanish,Malay": 0.24431818181818182, "Filipino,Spanish,Vietnamese": 0.23863636363636365, "Filipino,Chinese,Malay": 0.24431818181818182, "Filipino,Chinese,Vietnamese": 0.23295454545454544, "Filipino,Malay,Vietnamese": 0.23863636363636365, "Spanish,Chinese,Malay": 0.2840909090909091, "Spanish,Chinese,Vietnamese": 0.2784090909090909, "Spanish,Malay,Vietnamese": 0.2784090909090909, "Chinese,Malay,Vietnamese": 0.26136363636363635 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1875, "Indonesian,English,Filipino,Chinese": 0.1875, "Indonesian,English,Filipino,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese": 0.23863636363636365, "Indonesian,English,Spanish,Malay": 0.19886363636363635, "Indonesian,English,Spanish,Vietnamese": 0.22727272727272727, "Indonesian,English,Chinese,Malay": 0.2159090909090909, "Indonesian,English,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,English,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.1875, "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Chinese,Malay": 0.19318181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese,Malay": 0.23295454545454544, "Indonesian,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Chinese,Malay,Vietnamese": 0.19318181818181818, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.16477272727272727, "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, "English,Filipino,Chinese,Malay": 0.16477272727272727, "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "English,Filipino,Malay,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese,Malay": 0.19318181818181818, "English,Spanish,Chinese,Vietnamese": 0.19886363636363635, "English,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Chinese,Malay,Vietnamese": 0.16477272727272727, "Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.11931818181818182, "Indonesian,English,Spanish,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.125, "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "English,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 } }, "AC3_2": 0.4030935380265044, "AC3_3": 0.31222735432580656, "AC3_4": 0.24265311723031535, "AC3_5": 0.1903536485541224, "AC3_6": 0.15161104714689078, "AC3_7": 0.12212121209255435 } }, "sg_eval": { "prompt_1": { "accuracy": 0.39805825242718446 }, "prompt_2": { "accuracy": 0.3592233009708738 }, "prompt_3": { "accuracy": 0.4077669902912621 }, "prompt_4": { "accuracy": 0.4077669902912621 }, "prompt_5": { "accuracy": 0.39805825242718446 } }, "cn_eval": { "prompt_1": { "accuracy": 0.3333333333333333 }, "prompt_2": { "accuracy": 0.34285714285714286 }, "prompt_3": { "accuracy": 0.37142857142857144 }, "prompt_4": { "accuracy": 0.3238095238095238 }, "prompt_5": { "accuracy": 0.3238095238095238 } }, "us_eval": { "prompt_1": { "accuracy": 0.5046728971962616 }, "prompt_2": { "accuracy": 0.45794392523364486 }, "prompt_3": { "accuracy": 0.5420560747663551 }, "prompt_4": { "accuracy": 0.5327102803738317 }, "prompt_5": { "accuracy": 0.4672897196261682 } }, "ph_eval": { "prompt_1": { "accuracy": 0.39, "category_acc": { "brand": 0.3, "demographics": 0.0, "biology": 0.4, "history": 0.4, "literature": 0.3, "politics": 0.6, "culture": 0.5, "film": 0.3, "law": 0.2, "geography": 0.7 } }, "prompt_2": { "accuracy": 0.38, "category_acc": { "brand": 0.2, "demographics": 0.0, "biology": 0.4, "history": 0.3333333333333333, "literature": 0.2, "politics": 0.6, "culture": 0.6, "film": 0.4, "law": 0.2, "geography": 0.7 } }, "prompt_3": { "accuracy": 0.31, "category_acc": { "brand": 0.1, "demographics": 0.0, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.5, "culture": 0.4, "film": 0.5, "law": 0.2, "geography": 0.4 } }, "prompt_4": { "accuracy": 0.34, "category_acc": { "brand": 0.1, "demographics": 0.2, "biology": 0.5, "history": 0.26666666666666666, "literature": 0.1, "politics": 0.5, "culture": 0.4, "film": 0.4, "law": 0.3, "geography": 0.6 } }, "prompt_5": { "accuracy": 0.35, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.3333333333333333, "literature": 0.3, "politics": 0.5, "culture": 0.4, "film": 0.4, "law": 0.3, "geography": 0.3 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.16193972144329494 }, "prompt_2": { "bleu_score": 0.1736898619170894 }, "prompt_3": { "bleu_score": 0.1731736948698858 }, "prompt_4": { "bleu_score": 0.13471007416357628 }, "prompt_5": { "bleu_score": 0.1315596446656475 } }, "indommlu": { "prompt_1": { "accuracy": 0.3557647372988851, "category_acc": { "History": 0.3353413654618474, "Geography": 0.2714285714285714, "Lampungic": 0.38095238095238093, "Social science": 0.49248747913188645, "Balinese": 0.31210191082802546, "Makassarese": 0.27419354838709675, "Banjarese": 0.3055555555555556, "Chemistry": 0.2291970802919708, "Biology": 0.285207100591716, "Science": 0.40970072239422084, "Christian religion": 0.43283582089552236, "Art": 0.3860232945091514, "Islam religion": 0.4310099573257468, "Hindu religion": 0.4533333333333333, "Madurese": 0.2677966101694915, "Sport": 0.3581081081081081, "Indonesian language": 0.41344956413449563, "Physics": 0.2808080808080808, "Minangkabau culture": 0.35678391959798994, "Dayak language": 0.30275229357798167, "Sociology": 0.3125, "Economy": 0.2848360655737705, "Sundanese": 0.32757130509939497, "Javanese": 0.3094758064516129, "Civic education": 0.3834048640915594 } }, "prompt_2": { "accuracy": 0.34875492355965015, "category_acc": { "History": 0.321285140562249, "Geography": 0.26326530612244897, "Lampungic": 0.2653061224489796, "Social science": 0.4524207011686144, "Balinese": 0.2908704883227176, "Makassarese": 0.34408602150537637, "Banjarese": 0.3611111111111111, "Chemistry": 0.22043795620437956, "Biology": 0.2875739644970414, "Science": 0.39628482972136225, "Christian religion": 0.43781094527363185, "Art": 0.3910149750415973, "Islam religion": 0.406827880512091, "Hindu religion": 0.4, "Madurese": 0.33220338983050846, "Sport": 0.3716216216216216, "Indonesian language": 0.4078455790784558, "Physics": 0.27474747474747474, "Minangkabau culture": 0.36683417085427134, "Dayak language": 0.29357798165137616, "Sociology": 0.3064516129032258, "Economy": 0.2786885245901639, "Sundanese": 0.33189282627484873, "Javanese": 0.2862903225806452, "Civic education": 0.3791130185979971 } }, "prompt_3": { "accuracy": 0.35476333533613724, "category_acc": { "History": 0.35943775100401604, "Geography": 0.2816326530612245, "Lampungic": 0.29931972789115646, "Social science": 0.49081803005008345, "Balinese": 0.3078556263269639, "Makassarese": 0.3118279569892473, "Banjarese": 0.3680555555555556, "Chemistry": 0.22481751824817517, "Biology": 0.3076923076923077, "Science": 0.40350877192982454, "Christian religion": 0.42786069651741293, "Art": 0.37271214642262895, "Islam religion": 0.406827880512091, "Hindu religion": 0.41333333333333333, "Madurese": 0.2847457627118644, "Sport": 0.32432432432432434, "Indonesian language": 0.40255292652552926, "Physics": 0.30707070707070705, "Minangkabau culture": 0.38190954773869346, "Dayak language": 0.29357798165137616, "Sociology": 0.3125, "Economy": 0.29098360655737704, "Sundanese": 0.341400172860847, "Javanese": 0.29838709677419356, "Civic education": 0.38197424892703863 } }, "prompt_4": { "accuracy": 0.34822084251285135, "category_acc": { "History": 0.3253012048192771, "Geography": 0.2897959183673469, "Lampungic": 0.25170068027210885, "Social science": 0.4691151919866444, "Balinese": 0.3078556263269639, "Makassarese": 0.3010752688172043, "Banjarese": 0.3541666666666667, "Chemistry": 0.21021897810218979, "Biology": 0.30177514792899407, "Science": 0.38699690402476783, "Christian religion": 0.40298507462686567, "Art": 0.36938435940099834, "Islam religion": 0.39829302987197723, "Hindu religion": 0.41333333333333333, "Madurese": 0.30847457627118646, "Sport": 0.3108108108108108, "Indonesian language": 0.40006226650062265, "Physics": 0.29494949494949496, "Minangkabau culture": 0.3969849246231156, "Dayak language": 0.30275229357798167, "Sociology": 0.3185483870967742, "Economy": 0.27459016393442626, "Sundanese": 0.34053586862575624, "Javanese": 0.28125, "Civic education": 0.3977110157367668 } }, "prompt_5": { "accuracy": 0.3228519927899059, "category_acc": { "History": 0.30522088353413657, "Geography": 0.2653061224489796, "Lampungic": 0.23809523809523808, "Social science": 0.39065108514190316, "Balinese": 0.28874734607218683, "Makassarese": 0.34946236559139787, "Banjarese": 0.3472222222222222, "Chemistry": 0.22773722627737225, "Biology": 0.26745562130177514, "Science": 0.3333333333333333, "Christian religion": 0.38308457711442784, "Art": 0.3261231281198003, "Islam religion": 0.3684210526315789, "Hindu religion": 0.37333333333333335, "Madurese": 0.3050847457627119, "Sport": 0.3108108108108108, "Indonesian language": 0.3788916562889166, "Physics": 0.2808080808080808, "Minangkabau culture": 0.3869346733668342, "Dayak language": 0.26605504587155965, "Sociology": 0.3064516129032258, "Economy": 0.23565573770491804, "Sundanese": 0.2938634399308557, "Javanese": 0.29536290322580644, "Civic education": 0.34763948497854075 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.1344658289396944 }, "prompt_2": { "bleu_score": 0.19251961155388694 }, "prompt_3": { "bleu_score": 0.1954617717049563 }, "prompt_4": { "bleu_score": 0.1827612615161473 }, "prompt_5": { "bleu_score": 0.17386935444058352 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.08463019364617753 }, "prompt_2": { "bleu_score": 0.14858440530789044 }, "prompt_3": { "bleu_score": 0.14959642772587517 }, "prompt_4": { "bleu_score": 0.1385638325995254 }, "prompt_5": { "bleu_score": 0.14903962190314016 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.0675400082186485 }, "prompt_2": { "bleu_score": 0.1139916771789687 }, "prompt_3": { "bleu_score": 0.11287948125110897 }, "prompt_4": { "bleu_score": 0.10485948930289936 }, "prompt_5": { "bleu_score": 0.14472582688462934 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.10844882804173757 }, "prompt_2": { "bleu_score": 0.1665115420794729 }, "prompt_3": { "bleu_score": 0.16769944249339336 }, "prompt_4": { "bleu_score": 0.15952818432430357 }, "prompt_5": { "bleu_score": 0.1429123751624433 } }, "mmlu": { "prompt_1": { "accuracy": 0.4515752625437573 }, "prompt_2": { "accuracy": 0.47024504084014 }, "prompt_3": { "accuracy": 0.45274212368728123 }, "prompt_4": { "accuracy": 0.4340723453908985 }, "prompt_5": { "accuracy": 0.42123687281213534 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.4501251340722202, "category_acc": { "high_school_european_history": 0.6219512195121951, "business_ethics": 0.47474747474747475, "clinical_knowledge": 0.5037878787878788, "medical_genetics": 0.47474747474747475, "high_school_us_history": 0.6305418719211823, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.6271186440677966, "virology": 0.41818181818181815, "high_school_microeconomics": 0.4008438818565401, "econometrics": 0.23893805309734514, "college_computer_science": 0.32323232323232326, "high_school_biology": 0.4692556634304207, "abstract_algebra": 0.29292929292929293, "professional_accounting": 0.31316725978647686, "philosophy": 0.5, "professional_medicine": 0.3985239852398524, "nutrition": 0.47540983606557374, "global_facts": 0.45454545454545453, "machine_learning": 0.27927927927927926, "security_studies": 0.4180327868852459, "public_relations": 0.5321100917431193, "professional_psychology": 0.45499181669394434, "prehistory": 0.541795665634675, "anatomy": 0.4925373134328358, "human_sexuality": 0.4076923076923077, "college_medicine": 0.36627906976744184, "high_school_government_and_politics": 0.625, "college_chemistry": 0.1919191919191919, "logical_fallacies": 0.5123456790123457, "high_school_geography": 0.5786802030456852, "elementary_mathematics": 0.3952254641909814, "human_aging": 0.5855855855855856, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.6139705882352942, "formal_logic": 0.272, "high_school_statistics": 0.2837209302325581, "international_law": 0.6083333333333333, "high_school_mathematics": 0.20446096654275092, "high_school_computer_science": 0.40404040404040403, "conceptual_physics": 0.37606837606837606, "miscellaneous": 0.6879795396419437, "high_school_chemistry": 0.3069306930693069, "marketing": 0.6652360515021459, "professional_law": 0.3633398564905414, "management": 0.6176470588235294, "college_physics": 0.22772277227722773, "jurisprudence": 0.5700934579439252, "world_religions": 0.7058823529411765, "sociology": 0.645, "us_foreign_policy": 0.6161616161616161, "high_school_macroeconomics": 0.39588688946015427, "computer_security": 0.5353535353535354, "moral_scenarios": 0.23378076062639822, "moral_disputes": 0.47246376811594204, "electrical_engineering": 0.4722222222222222, "astronomy": 0.4503311258278146, "college_biology": 0.4965034965034965 } }, "prompt_2": { "accuracy": 0.45055416517697533, "category_acc": { "high_school_european_history": 0.5792682926829268, "business_ethics": 0.5151515151515151, "clinical_knowledge": 0.4583333333333333, "medical_genetics": 0.46464646464646464, "high_school_us_history": 0.6403940886699507, "high_school_physics": 0.3, "high_school_world_history": 0.6271186440677966, "virology": 0.40606060606060607, "high_school_microeconomics": 0.39662447257383965, "econometrics": 0.24778761061946902, "college_computer_science": 0.3838383838383838, "high_school_biology": 0.48220064724919093, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.33451957295373663, "philosophy": 0.5193548387096775, "professional_medicine": 0.4059040590405904, "nutrition": 0.49836065573770494, "global_facts": 0.43434343434343436, "machine_learning": 0.3333333333333333, "security_studies": 0.4180327868852459, "public_relations": 0.5412844036697247, "professional_psychology": 0.4320785597381342, "prehistory": 0.5356037151702786, "anatomy": 0.4701492537313433, "human_sexuality": 0.43846153846153846, "college_medicine": 0.3953488372093023, "high_school_government_and_politics": 0.6770833333333334, "college_chemistry": 0.25252525252525254, "logical_fallacies": 0.5617283950617284, "high_school_geography": 0.5532994923857868, "elementary_mathematics": 0.3660477453580902, "human_aging": 0.545045045045045, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.6158088235294118, "formal_logic": 0.272, "high_school_statistics": 0.2744186046511628, "international_law": 0.5833333333333334, "high_school_mathematics": 0.23048327137546468, "high_school_computer_science": 0.3838383838383838, "conceptual_physics": 0.3803418803418803, "miscellaneous": 0.6815856777493606, "high_school_chemistry": 0.33663366336633666, "marketing": 0.6952789699570815, "professional_law": 0.3639921722113503, "management": 0.5882352941176471, "college_physics": 0.25742574257425743, "jurisprudence": 0.5327102803738317, "world_religions": 0.6941176470588235, "sociology": 0.625, "us_foreign_policy": 0.6464646464646465, "high_school_macroeconomics": 0.37017994858611825, "computer_security": 0.5454545454545454, "moral_scenarios": 0.24720357941834453, "moral_disputes": 0.4927536231884058, "electrical_engineering": 0.4097222222222222, "astronomy": 0.39072847682119205, "college_biology": 0.5174825174825175 } }, "prompt_3": { "accuracy": 0.43425098319628175, "category_acc": { "high_school_european_history": 0.6036585365853658, "business_ethics": 0.45454545454545453, "clinical_knowledge": 0.4696969696969697, "medical_genetics": 0.3838383838383838, "high_school_us_history": 0.6403940886699507, "high_school_physics": 0.24666666666666667, "high_school_world_history": 0.614406779661017, "virology": 0.41818181818181815, "high_school_microeconomics": 0.379746835443038, "econometrics": 0.2920353982300885, "college_computer_science": 0.3838383838383838, "high_school_biology": 0.42071197411003236, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.33807829181494664, "philosophy": 0.4935483870967742, "professional_medicine": 0.3948339483394834, "nutrition": 0.45901639344262296, "global_facts": 0.42424242424242425, "machine_learning": 0.3153153153153153, "security_studies": 0.4139344262295082, "public_relations": 0.47706422018348627, "professional_psychology": 0.41734860883797054, "prehistory": 0.5294117647058824, "anatomy": 0.44776119402985076, "human_sexuality": 0.4230769230769231, "college_medicine": 0.3313953488372093, "high_school_government_and_politics": 0.625, "college_chemistry": 0.2222222222222222, "logical_fallacies": 0.5185185185185185, "high_school_geography": 0.49238578680203043, "elementary_mathematics": 0.4244031830238727, "human_aging": 0.5225225225225225, "college_mathematics": 0.2727272727272727, "high_school_psychology": 0.5772058823529411, "formal_logic": 0.328, "high_school_statistics": 0.31627906976744186, "international_law": 0.575, "high_school_mathematics": 0.20817843866171004, "high_school_computer_science": 0.41414141414141414, "conceptual_physics": 0.3547008547008547, "miscellaneous": 0.6547314578005116, "high_school_chemistry": 0.3316831683168317, "marketing": 0.6437768240343348, "professional_law": 0.35290280495759946, "management": 0.5686274509803921, "college_physics": 0.19801980198019803, "jurisprudence": 0.5607476635514018, "world_religions": 0.7, "sociology": 0.58, "us_foreign_policy": 0.6363636363636364, "high_school_macroeconomics": 0.37789203084832906, "computer_security": 0.48484848484848486, "moral_scenarios": 0.2516778523489933, "moral_disputes": 0.45507246376811594, "electrical_engineering": 0.3611111111111111, "astronomy": 0.3841059602649007, "college_biology": 0.3706293706293706 } }, "prompt_4": { "accuracy": 0.42710046478369684, "category_acc": { "high_school_european_history": 0.6158536585365854, "business_ethics": 0.3838383838383838, "clinical_knowledge": 0.44696969696969696, "medical_genetics": 0.42424242424242425, "high_school_us_history": 0.6108374384236454, "high_school_physics": 0.3, "high_school_world_history": 0.6101694915254238, "virology": 0.36363636363636365, "high_school_microeconomics": 0.3628691983122363, "econometrics": 0.25663716814159293, "college_computer_science": 0.29292929292929293, "high_school_biology": 0.45307443365695793, "abstract_algebra": 0.26262626262626265, "professional_accounting": 0.30604982206405695, "philosophy": 0.4806451612903226, "professional_medicine": 0.3948339483394834, "nutrition": 0.4491803278688525, "global_facts": 0.5050505050505051, "machine_learning": 0.2702702702702703, "security_studies": 0.42213114754098363, "public_relations": 0.5596330275229358, "professional_psychology": 0.397708674304419, "prehistory": 0.47987616099071206, "anatomy": 0.43283582089552236, "human_sexuality": 0.4461538461538462, "college_medicine": 0.31976744186046513, "high_school_government_and_politics": 0.6197916666666666, "college_chemistry": 0.29292929292929293, "logical_fallacies": 0.4876543209876543, "high_school_geography": 0.47715736040609136, "elementary_mathematics": 0.4244031830238727, "human_aging": 0.4774774774774775, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.5625, "formal_logic": 0.328, "high_school_statistics": 0.29767441860465116, "international_law": 0.5916666666666667, "high_school_mathematics": 0.26022304832713755, "high_school_computer_science": 0.46464646464646464, "conceptual_physics": 0.32905982905982906, "miscellaneous": 0.6534526854219949, "high_school_chemistry": 0.28217821782178215, "marketing": 0.5965665236051502, "professional_law": 0.3476842791911285, "management": 0.5882352941176471, "college_physics": 0.26732673267326734, "jurisprudence": 0.514018691588785, "world_religions": 0.6705882352941176, "sociology": 0.525, "us_foreign_policy": 0.6363636363636364, "high_school_macroeconomics": 0.38303341902313626, "computer_security": 0.5151515151515151, "moral_scenarios": 0.2371364653243848, "moral_disputes": 0.45507246376811594, "electrical_engineering": 0.4027777777777778, "astronomy": 0.37748344370860926, "college_biology": 0.4405594405594406 } }, "prompt_5": { "accuracy": 0.42617089739006075, "category_acc": { "high_school_european_history": 0.5975609756097561, "business_ethics": 0.45454545454545453, "clinical_knowledge": 0.44696969696969696, "medical_genetics": 0.41414141414141414, "high_school_us_history": 0.5714285714285714, "high_school_physics": 0.31333333333333335, "high_school_world_history": 0.5677966101694916, "virology": 0.3696969696969697, "high_school_microeconomics": 0.3628691983122363, "econometrics": 0.23893805309734514, "college_computer_science": 0.2727272727272727, "high_school_biology": 0.47249190938511326, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.27402135231316727, "philosophy": 0.4870967741935484, "professional_medicine": 0.3948339483394834, "nutrition": 0.4557377049180328, "global_facts": 0.5050505050505051, "machine_learning": 0.2702702702702703, "security_studies": 0.36885245901639346, "public_relations": 0.46788990825688076, "professional_psychology": 0.4320785597381342, "prehistory": 0.4984520123839009, "anatomy": 0.4552238805970149, "human_sexuality": 0.3769230769230769, "college_medicine": 0.38953488372093026, "high_school_government_and_politics": 0.6145833333333334, "college_chemistry": 0.30303030303030304, "logical_fallacies": 0.47530864197530864, "high_school_geography": 0.4720812182741117, "elementary_mathematics": 0.35809018567639256, "human_aging": 0.4864864864864865, "college_mathematics": 0.26262626262626265, "high_school_psychology": 0.5808823529411765, "formal_logic": 0.304, "high_school_statistics": 0.33488372093023255, "international_law": 0.5833333333333334, "high_school_mathematics": 0.30111524163568776, "high_school_computer_science": 0.3838383838383838, "conceptual_physics": 0.3974358974358974, "miscellaneous": 0.6240409207161125, "high_school_chemistry": 0.30198019801980197, "marketing": 0.5708154506437768, "professional_law": 0.34898891063274623, "management": 0.5588235294117647, "college_physics": 0.31683168316831684, "jurisprudence": 0.4953271028037383, "world_religions": 0.6470588235294118, "sociology": 0.555, "us_foreign_policy": 0.5959595959595959, "high_school_macroeconomics": 0.35732647814910024, "computer_security": 0.5050505050505051, "moral_scenarios": 0.25727069351230425, "moral_disputes": 0.43768115942028984, "electrical_engineering": 0.4166666666666667, "astronomy": 0.423841059602649, "college_biology": 0.46853146853146854 } } }, "c_eval": { "prompt_1": { "accuracy": 0.35438335809806837 }, "prompt_2": { "accuracy": 0.3350668647845468 }, "prompt_3": { "accuracy": 0.3462109955423477 }, "prompt_4": { "accuracy": 0.3462109955423477 }, "prompt_5": { "accuracy": 0.34175334323922735 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.3412204234122042, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.3333333333333333, "computer_architecture": 0.34615384615384615, "college_programming": 0.38095238095238093, "college_physics": 0.4583333333333333, "college_chemistry": 0.13793103448275862, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.38095238095238093, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.4827586206896552, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.38461538461538464, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.45, "business_administration": 0.4473684210526316, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.4117647058823529, "teacher_qualification": 0.4489795918367347, "high_school_politics": 0.4583333333333333, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.5, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.2857142857142857, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.2962962962962963, "law": 0.2413793103448276, "chinese_language_and_literature": 0.17857142857142858, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.4411764705882353, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.48, "middle_school_history": 0.4444444444444444, "civil_servant": 0.36538461538461536, "sports_science": 0.375, "plant_protection": 0.18518518518518517, "basic_medicine": 0.25, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.35185185185185186, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.2777777777777778, "physician": 0.24074074074074073 } }, "prompt_2": { "accuracy": 0.3343711083437111, "category_acc": { "computer_network": 0.375, "operating_system": 0.2916666666666667, "computer_architecture": 0.34615384615384615, "college_programming": 0.3333333333333333, "college_physics": 0.5, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.16666666666666666, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.4827586206896552, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.125, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.16666666666666666, "middle_school_mathematics": 0.125, "middle_school_biology": 0.2692307692307692, "middle_school_physics": 0.375, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.4, "business_administration": 0.4473684210526316, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.41379310344827586, "education_science": 0.4117647058823529, "teacher_qualification": 0.46938775510204084, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.5, "logic": 0.37037037037037035, "law": 0.2413793103448276, "chinese_language_and_literature": 0.25, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.2916666666666667, "high_school_history": 0.4, "middle_school_history": 0.5185185185185185, "civil_servant": 0.36538461538461536, "sports_science": 0.2916666666666667, "plant_protection": 0.4444444444444444, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.25925925925925924, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.2777777777777778, "physician": 0.3148148148148148 } }, "prompt_3": { "accuracy": 0.34184308841843086, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.3333333333333333, "computer_architecture": 0.34615384615384615, "college_programming": 0.3333333333333333, "college_physics": 0.3333333333333333, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.25, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.4827586206896552, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.25, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.3333333333333333, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.4, "business_administration": 0.3684210526315789, "marxism": 0.5833333333333334, "mao_zedong_thought": 0.5172413793103449, "education_science": 0.38235294117647056, "teacher_qualification": 0.5306122448979592, "high_school_politics": 0.5, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.4230769230769231, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.32142857142857145, "ideological_and_moral_cultivation": 0.4583333333333333, "logic": 0.25925925925925924, "law": 0.2413793103448276, "chinese_language_and_literature": 0.17857142857142858, "art_studies": 0.42105263157894735, "professional_tour_guide": 0.4117647058823529, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.4, "middle_school_history": 0.4074074074074074, "civil_servant": 0.36538461538461536, "sports_science": 0.5, "plant_protection": 0.25925925925925924, "basic_medicine": 0.25, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.39215686274509803, "accountant": 0.3888888888888889, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.2222222222222222, "physician": 0.2777777777777778 } }, "prompt_4": { "accuracy": 0.3163138231631382, "category_acc": { "computer_network": 0.16666666666666666, "operating_system": 0.2916666666666667, "computer_architecture": 0.3076923076923077, "college_programming": 0.47619047619047616, "college_physics": 0.375, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.125, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.4827586206896552, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.125, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.20833333333333334, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.38333333333333336, "business_administration": 0.3684210526315789, "marxism": 0.5416666666666666, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.29411764705882354, "teacher_qualification": 0.40816326530612246, "high_school_politics": 0.5, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.4230769230769231, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.35714285714285715, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.4444444444444444, "law": 0.3103448275862069, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.47368421052631576, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.25, "high_school_chinese": 0.25, "high_school_history": 0.32, "middle_school_history": 0.3333333333333333, "civil_servant": 0.3269230769230769, "sports_science": 0.375, "plant_protection": 0.2222222222222222, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.2222222222222222, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.2037037037037037, "physician": 0.2222222222222222 } }, "prompt_5": { "accuracy": 0.33872976338729766, "category_acc": { "computer_network": 0.4166666666666667, "operating_system": 0.16666666666666666, "computer_architecture": 0.38461538461538464, "college_programming": 0.4523809523809524, "college_physics": 0.3333333333333333, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.375, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.35714285714285715, "metrology_engineer": 0.4827586206896552, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.25, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.16666666666666666, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.2692307692307692, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.2, "veterinary_medicine": 0.17857142857142858, "college_economics": 0.35, "business_administration": 0.39473684210526316, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.5517241379310345, "education_science": 0.4411764705882353, "teacher_qualification": 0.42857142857142855, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.4230769230769231, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.39285714285714285, "ideological_and_moral_cultivation": 0.5, "logic": 0.4074074074074074, "law": 0.27586206896551724, "chinese_language_and_literature": 0.25, "art_studies": 0.4473684210526316, "professional_tour_guide": 0.5, "legal_professional": 0.17857142857142858, "high_school_chinese": 0.5416666666666666, "high_school_history": 0.56, "middle_school_history": 0.4074074074074074, "civil_servant": 0.28846153846153844, "sports_science": 0.25, "plant_protection": 0.3333333333333333, "basic_medicine": 0.2916666666666667, "clinical_medicine": 0.25925925925925924, "urban_and_rural_planner": 0.35294117647058826, "accountant": 0.2962962962962963, "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.2222222222222222, "physician": 0.25925925925925924 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.3046594982078853 }, "prompt_2": { "accuracy": 0.2974910394265233 }, "prompt_3": { "accuracy": 0.26881720430107525 }, "prompt_4": { "accuracy": 0.2974910394265233 }, "prompt_5": { "accuracy": 0.3046594982078853 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.33103091003280954, "category_acc": { "agronomy": 0.33727810650887574, "anatomy": 0.25675675675675674, "ancient_chinese": 0.2804878048780488, "arts": 0.325, "astronomy": 0.3151515151515151, "business_ethics": 0.36363636363636365, "chinese_civil_service_exam": 0.29375, "chinese_driving_rule": 0.42748091603053434, "chinese_food_culture": 0.2867647058823529, "chinese_foreign_policy": 0.4205607476635514, "chinese_history": 0.43343653250773995, "chinese_literature": 0.3137254901960784, "chinese_teacher_qualification": 0.4022346368715084, "clinical_knowledge": 0.270042194092827, "college_actuarial_science": 0.24528301886792453, "college_education": 0.3364485981308411, "college_engineering_hydrology": 0.3584905660377358, "college_law": 0.28703703703703703, "college_mathematics": 0.19047619047619047, "college_medical_statistics": 0.33962264150943394, "college_medicine": 0.30036630036630035, "computer_science": 0.30392156862745096, "computer_security": 0.30409356725146197, "conceptual_physics": 0.35374149659863946, "construction_project_management": 0.35251798561151076, "economics": 0.3584905660377358, "education": 0.3312883435582822, "electrical_engineering": 0.36627906976744184, "elementary_chinese": 0.2698412698412698, "elementary_commonsense": 0.31313131313131315, "elementary_information_and_technology": 0.4369747899159664, "elementary_mathematics": 0.2565217391304348, "ethnology": 0.31851851851851853, "food_science": 0.34965034965034963, "genetics": 0.30113636363636365, "global_facts": 0.31543624161073824, "high_school_biology": 0.28994082840236685, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.3474576271186441, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.3090909090909091, "high_school_politics": 0.35664335664335667, "human_sexuality": 0.36507936507936506, "international_law": 0.31351351351351353, "journalism": 0.3488372093023256, "jurisprudence": 0.35036496350364965, "legal_and_moral_basis": 0.5841121495327103, "logical": 0.35772357723577236, "machine_learning": 0.20491803278688525, "management": 0.34285714285714286, "marketing": 0.39444444444444443, "marxist_theory": 0.4021164021164021, "modern_chinese": 0.25, "nutrition": 0.3103448275862069, "philosophy": 0.45714285714285713, "professional_accounting": 0.32, "professional_law": 0.33649289099526064, "professional_medicine": 0.23670212765957446, "professional_psychology": 0.3448275862068966, "public_relations": 0.3735632183908046, "security_study": 0.28888888888888886, "sociology": 0.3141592920353982, "sports_science": 0.34545454545454546, "traditional_chinese_medicine": 0.2594594594594595, "virology": 0.33136094674556216, "world_history": 0.32919254658385094, "world_religions": 0.4125 } }, "prompt_2": { "accuracy": 0.3236919357623899, "category_acc": { "agronomy": 0.26627218934911245, "anatomy": 0.28378378378378377, "ancient_chinese": 0.2804878048780488, "arts": 0.35625, "astronomy": 0.3090909090909091, "business_ethics": 0.3349282296650718, "chinese_civil_service_exam": 0.2875, "chinese_driving_rule": 0.366412213740458, "chinese_food_culture": 0.3088235294117647, "chinese_foreign_policy": 0.308411214953271, "chinese_history": 0.4148606811145511, "chinese_literature": 0.29901960784313725, "chinese_teacher_qualification": 0.3743016759776536, "clinical_knowledge": 0.29957805907172996, "college_actuarial_science": 0.2358490566037736, "college_education": 0.35514018691588783, "college_engineering_hydrology": 0.330188679245283, "college_law": 0.23148148148148148, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.3018867924528302, "college_medicine": 0.2967032967032967, "computer_science": 0.3235294117647059, "computer_security": 0.3157894736842105, "conceptual_physics": 0.30612244897959184, "construction_project_management": 0.22302158273381295, "economics": 0.33962264150943394, "education": 0.3374233128834356, "electrical_engineering": 0.36627906976744184, "elementary_chinese": 0.2619047619047619, "elementary_commonsense": 0.29292929292929293, "elementary_information_and_technology": 0.4579831932773109, "elementary_mathematics": 0.30869565217391304, "ethnology": 0.34814814814814815, "food_science": 0.3776223776223776, "genetics": 0.26136363636363635, "global_facts": 0.3422818791946309, "high_school_biology": 0.27218934911242604, "high_school_chemistry": 0.3333333333333333, "high_school_geography": 0.3305084745762712, "high_school_mathematics": 0.24390243902439024, "high_school_physics": 0.3, "high_school_politics": 0.32167832167832167, "human_sexuality": 0.2857142857142857, "international_law": 0.32432432432432434, "journalism": 0.3372093023255814, "jurisprudence": 0.35036496350364965, "legal_and_moral_basis": 0.5280373831775701, "logical": 0.3089430894308943, "machine_learning": 0.23770491803278687, "management": 0.3333333333333333, "marketing": 0.4, "marxist_theory": 0.3544973544973545, "modern_chinese": 0.22413793103448276, "nutrition": 0.2413793103448276, "philosophy": 0.4, "professional_accounting": 0.3485714285714286, "professional_law": 0.32701421800947866, "professional_medicine": 0.24202127659574468, "professional_psychology": 0.35344827586206895, "public_relations": 0.3793103448275862, "security_study": 0.32592592592592595, "sociology": 0.33185840707964603, "sports_science": 0.36363636363636365, "traditional_chinese_medicine": 0.2972972972972973, "virology": 0.31952662721893493, "world_history": 0.3167701863354037, "world_religions": 0.375 } }, "prompt_3": { "accuracy": 0.3265411845967881, "category_acc": { "agronomy": 0.31952662721893493, "anatomy": 0.25, "ancient_chinese": 0.22560975609756098, "arts": 0.325, "astronomy": 0.3151515151515151, "business_ethics": 0.3684210526315789, "chinese_civil_service_exam": 0.2875, "chinese_driving_rule": 0.42748091603053434, "chinese_food_culture": 0.3014705882352941, "chinese_foreign_policy": 0.4205607476635514, "chinese_history": 0.42724458204334365, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.4022346368715084, "clinical_knowledge": 0.27848101265822783, "college_actuarial_science": 0.2169811320754717, "college_education": 0.38317757009345793, "college_engineering_hydrology": 0.3584905660377358, "college_law": 0.25925925925925924, "college_mathematics": 0.24761904761904763, "college_medical_statistics": 0.27358490566037735, "college_medicine": 0.2893772893772894, "computer_science": 0.3137254901960784, "computer_security": 0.3508771929824561, "conceptual_physics": 0.3333333333333333, "construction_project_management": 0.2805755395683453, "economics": 0.34591194968553457, "education": 0.38650306748466257, "electrical_engineering": 0.31976744186046513, "elementary_chinese": 0.25396825396825395, "elementary_commonsense": 0.3181818181818182, "elementary_information_and_technology": 0.3865546218487395, "elementary_mathematics": 0.27391304347826084, "ethnology": 0.32592592592592595, "food_science": 0.34965034965034963, "genetics": 0.2556818181818182, "global_facts": 0.3221476510067114, "high_school_biology": 0.2603550295857988, "high_school_chemistry": 0.2878787878787879, "high_school_geography": 0.2796610169491525, "high_school_mathematics": 0.1951219512195122, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.32167832167832167, "human_sexuality": 0.35714285714285715, "international_law": 0.31351351351351353, "journalism": 0.3546511627906977, "jurisprudence": 0.3284671532846715, "legal_and_moral_basis": 0.5841121495327103, "logical": 0.3333333333333333, "machine_learning": 0.2459016393442623, "management": 0.3238095238095238, "marketing": 0.37777777777777777, "marxist_theory": 0.43386243386243384, "modern_chinese": 0.2413793103448276, "nutrition": 0.2896551724137931, "philosophy": 0.41904761904761906, "professional_accounting": 0.3314285714285714, "professional_law": 0.33649289099526064, "professional_medicine": 0.23670212765957446, "professional_psychology": 0.375, "public_relations": 0.3850574712643678, "security_study": 0.3037037037037037, "sociology": 0.336283185840708, "sports_science": 0.37575757575757573, "traditional_chinese_medicine": 0.24864864864864866, "virology": 0.31952662721893493, "world_history": 0.35403726708074534, "world_religions": 0.43125 } }, "prompt_4": { "accuracy": 0.31445346226903814, "category_acc": { "agronomy": 0.26627218934911245, "anatomy": 0.28378378378378377, "ancient_chinese": 0.23170731707317074, "arts": 0.33125, "astronomy": 0.3393939393939394, "business_ethics": 0.33014354066985646, "chinese_civil_service_exam": 0.2625, "chinese_driving_rule": 0.31297709923664124, "chinese_food_culture": 0.2867647058823529, "chinese_foreign_policy": 0.35514018691588783, "chinese_history": 0.38699690402476783, "chinese_literature": 0.3137254901960784, "chinese_teacher_qualification": 0.3407821229050279, "clinical_knowledge": 0.29535864978902954, "college_actuarial_science": 0.20754716981132076, "college_education": 0.32710280373831774, "college_engineering_hydrology": 0.3113207547169811, "college_law": 0.2037037037037037, "college_mathematics": 0.22857142857142856, "college_medical_statistics": 0.3018867924528302, "college_medicine": 0.31868131868131866, "computer_science": 0.3235294117647059, "computer_security": 0.2807017543859649, "conceptual_physics": 0.2857142857142857, "construction_project_management": 0.26618705035971224, "economics": 0.3710691823899371, "education": 0.3067484662576687, "electrical_engineering": 0.3023255813953488, "elementary_chinese": 0.2698412698412698, "elementary_commonsense": 0.30808080808080807, "elementary_information_and_technology": 0.42857142857142855, "elementary_mathematics": 0.26956521739130435, "ethnology": 0.32592592592592595, "food_science": 0.34965034965034963, "genetics": 0.24431818181818182, "global_facts": 0.3288590604026846, "high_school_biology": 0.28402366863905326, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.3389830508474576, "high_school_mathematics": 0.2682926829268293, "high_school_physics": 0.3090909090909091, "high_school_politics": 0.34265734265734266, "human_sexuality": 0.3253968253968254, "international_law": 0.35135135135135137, "journalism": 0.3313953488372093, "jurisprudence": 0.32116788321167883, "legal_and_moral_basis": 0.5420560747663551, "logical": 0.2764227642276423, "machine_learning": 0.2459016393442623, "management": 0.319047619047619, "marketing": 0.37222222222222223, "marxist_theory": 0.4074074074074074, "modern_chinese": 0.2672413793103448, "nutrition": 0.27586206896551724, "philosophy": 0.38095238095238093, "professional_accounting": 0.2914285714285714, "professional_law": 0.33649289099526064, "professional_medicine": 0.26063829787234044, "professional_psychology": 0.3275862068965517, "public_relations": 0.3275862068965517, "security_study": 0.31851851851851853, "sociology": 0.3185840707964602, "sports_science": 0.2787878787878788, "traditional_chinese_medicine": 0.23783783783783785, "virology": 0.30177514792899407, "world_history": 0.32298136645962733, "world_religions": 0.3875 } }, "prompt_5": { "accuracy": 0.32593679848040064, "category_acc": { "agronomy": 0.28402366863905326, "anatomy": 0.2702702702702703, "ancient_chinese": 0.23780487804878048, "arts": 0.36875, "astronomy": 0.3393939393939394, "business_ethics": 0.35406698564593303, "chinese_civil_service_exam": 0.28125, "chinese_driving_rule": 0.4122137404580153, "chinese_food_culture": 0.3161764705882353, "chinese_foreign_policy": 0.3177570093457944, "chinese_history": 0.4024767801857585, "chinese_literature": 0.29901960784313725, "chinese_teacher_qualification": 0.3854748603351955, "clinical_knowledge": 0.27848101265822783, "college_actuarial_science": 0.2169811320754717, "college_education": 0.29906542056074764, "college_engineering_hydrology": 0.3490566037735849, "college_law": 0.25925925925925924, "college_mathematics": 0.2, "college_medical_statistics": 0.36792452830188677, "college_medicine": 0.315018315018315, "computer_science": 0.3382352941176471, "computer_security": 0.29239766081871343, "conceptual_physics": 0.3333333333333333, "construction_project_management": 0.37410071942446044, "economics": 0.389937106918239, "education": 0.3128834355828221, "electrical_engineering": 0.3430232558139535, "elementary_chinese": 0.2619047619047619, "elementary_commonsense": 0.32323232323232326, "elementary_information_and_technology": 0.4117647058823529, "elementary_mathematics": 0.3347826086956522, "ethnology": 0.34074074074074073, "food_science": 0.36363636363636365, "genetics": 0.23295454545454544, "global_facts": 0.2550335570469799, "high_school_biology": 0.23076923076923078, "high_school_chemistry": 0.24242424242424243, "high_school_geography": 0.3389830508474576, "high_school_mathematics": 0.24390243902439024, "high_school_physics": 0.33636363636363636, "high_school_politics": 0.3146853146853147, "human_sexuality": 0.30158730158730157, "international_law": 0.32432432432432434, "journalism": 0.3372093023255814, "jurisprudence": 0.340632603406326, "legal_and_moral_basis": 0.5607476635514018, "logical": 0.34959349593495936, "machine_learning": 0.28688524590163933, "management": 0.36666666666666664, "marketing": 0.38333333333333336, "marxist_theory": 0.3862433862433862, "modern_chinese": 0.2413793103448276, "nutrition": 0.32413793103448274, "philosophy": 0.44761904761904764, "professional_accounting": 0.26857142857142857, "professional_law": 0.3412322274881517, "professional_medicine": 0.23404255319148937, "professional_psychology": 0.3146551724137931, "public_relations": 0.3505747126436782, "security_study": 0.34074074074074073, "sociology": 0.3407079646017699, "sports_science": 0.37575757575757573, "traditional_chinese_medicine": 0.2702702702702703, "virology": 0.33136094674556216, "world_history": 0.3416149068322981, "world_religions": 0.3875 } } }, "zbench": { "prompt_1": { "accuracy": 0.30303030303030304 }, "prompt_2": { "accuracy": 0.24242424242424243 }, "prompt_3": { "accuracy": 0.2727272727272727 }, "prompt_4": { "accuracy": 0.30303030303030304 }, "prompt_5": { "accuracy": 0.2727272727272727 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.3886363636363636 }, "prompt_2": { "accuracy": 0.3886363636363636 }, "prompt_3": { "accuracy": 0.4113636363636364 }, "prompt_4": { "accuracy": 0.4113636363636364 }, "prompt_5": { "accuracy": 0.4068181818181818 } }, "ocnli": { "prompt_1": { "accuracy": 0.34983050847457625 }, "prompt_2": { "accuracy": 0.3494915254237288 }, "prompt_3": { "accuracy": 0.34440677966101696 }, "prompt_4": { "accuracy": 0.35084745762711866 }, "prompt_5": { "accuracy": 0.3501694915254237 } }, "c3": { "prompt_1": { "accuracy": 0.6353777112939416 }, "prompt_2": { "accuracy": 0.6245325355272999 }, "prompt_3": { "accuracy": 0.6357516828721017 }, "prompt_4": { "accuracy": 0.637247569184742 }, "prompt_5": { "accuracy": 0.6264023934181002 } }, "dream": { "prompt_1": { "accuracy": 0.7457128858402744 }, "prompt_2": { "accuracy": 0.7442430181283685 }, "prompt_3": { "accuracy": 0.7560019598236158 }, "prompt_4": { "accuracy": 0.7457128858402744 }, "prompt_5": { "accuracy": 0.7648211660950515 } }, "samsum": { "prompt_1": { "rouge1": 0.30429037973491263, "rouge2": 0.11166560635819005, "rougeL": 0.2288226789920716, "avg_rouge": 0.21492622169505812 }, "prompt_2": { "rouge1": 0.3201013255766478, "rouge2": 0.12129474572573874, "rougeL": 0.24126673160420492, "avg_rouge": 0.22755426763553047 }, "prompt_3": { "rouge1": 0.3229019271832191, "rouge2": 0.11584270451474661, "rougeL": 0.23985168151642056, "avg_rouge": 0.22619877107146205 }, "prompt_4": { "rouge1": 0.32156871672144266, "rouge2": 0.11707761581573511, "rougeL": 0.24020166411072763, "avg_rouge": 0.2262826655493018 }, "prompt_5": { "rouge1": 0.3323899982736654, "rouge2": 0.11772618412195887, "rougeL": 0.24821670371992166, "avg_rouge": 0.23277762870518195 } }, "dialogsum": { "prompt_1": { "rouge1": 0.21951377873070885, "rouge2": 0.054306230540633486, "rougeL": 0.15617467192439893, "avg_rouge": 0.14333156039858042 }, "prompt_2": { "rouge1": 0.22295085388446095, "rouge2": 0.05554955905799761, "rougeL": 0.15911825564524076, "avg_rouge": 0.14587288952923314 }, "prompt_3": { "rouge1": 0.2225600349636942, "rouge2": 0.05514136914149386, "rougeL": 0.15803083808974638, "avg_rouge": 0.14524408073164483 }, "prompt_4": { "rouge1": 0.21699651938260173, "rouge2": 0.05393280959156006, "rougeL": 0.15328420391743472, "avg_rouge": 0.1414045109638655 }, "prompt_5": { "rouge1": 0.22324097016418196, "rouge2": 0.056535321282525815, "rougeL": 0.16041089625980098, "avg_rouge": 0.14672906256883625 } }, "sst2": { "prompt_1": { "accuracy": 0.6892201834862385 }, "prompt_2": { "accuracy": 0.7190366972477065 }, "prompt_3": { "accuracy": 0.7155963302752294 }, "prompt_4": { "accuracy": 0.6295871559633027 }, "prompt_5": { "accuracy": 0.7213302752293578 } }, "cola": { "prompt_1": { "accuracy": 0.7583892617449665 }, "prompt_2": { "accuracy": 0.74784276126558 }, "prompt_3": { "accuracy": 0.7574304889741131 }, "prompt_4": { "accuracy": 0.7737296260786194 }, "prompt_5": { "accuracy": 0.7248322147651006 } }, "qqp": { "prompt_1": { "accuracy": 0.5165 }, "prompt_2": { "accuracy": 0.604 }, "prompt_3": { "accuracy": 0.5035 }, "prompt_4": { "accuracy": 0.523 }, "prompt_5": { "accuracy": 0.584 } }, "mnli": { "prompt_1": { "accuracy": 0.2 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.2 } }, "qnli": { "prompt_1": { "accuracy": 0.4 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } }, "rte": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.7 } }, "mrpc": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.6 }, "prompt_5": { "accuracy": 0.4 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "gpt4_1106_preview": { "model_size": "NA", "model_link": "https://openai.com/blog/chatgpt", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9535714285714285, "language_acc": { "Vietnamese": 0.9546218487394958, "Spanish": 0.9588235294117647, "English": 0.9588235294117647, "Chinese": 0.9420168067226891 }, "consistency_score_2": 0.953921568627451, "consistency_score_3": 0.9315126050420168, "consistency_score_4": 0.9117647058823529, "detailed_consistency_score": { "2_combine": { "Vietnamese,Spanish": 0.9596638655462185, "Vietnamese,English": 0.965546218487395, "Vietnamese,Chinese": 0.9436974789915966, "Spanish,English": 0.9638655462184874, "Spanish,Chinese": 0.9445378151260504, "English,Chinese": 0.946218487394958 }, "3_combine": { "Vietnamese,Spanish,English": 0.9453781512605042, "Vietnamese,Spanish,Chinese": 0.9243697478991597, "Vietnamese,English,Chinese": 0.9285714285714286, "Spanish,English,Chinese": 0.9277310924369748 }, "4_combine": { "Vietnamese,Spanish,English,Chinese": 0.9117647058823529 } }, "AC3_2": 0.9537464664135267, "AC3_3": 0.9424129530951566, "AC3_4": 0.9321995719739016 }, "prompt_2": { "overall_acc": 0.9527310924369747, "language_acc": { "Vietnamese": 0.9521008403361344, "Spanish": 0.9529411764705882, "English": 0.9563025210084034, "Chinese": 0.9495798319327731 }, "consistency_score_2": 0.9551820728291317, "consistency_score_3": 0.934453781512605, "consistency_score_4": 0.9193277310924369, "detailed_consistency_score": { "2_combine": { "Vietnamese,Spanish": 0.9537815126050421, "Vietnamese,English": 0.9680672268907563, "Vietnamese,Chinese": 0.9453781512605042, "Spanish,English": 0.9663865546218487, "Spanish,Chinese": 0.9445378151260504, "English,Chinese": 0.9529411764705882 }, "3_combine": { "Vietnamese,Spanish,English": 0.9445378151260504, "Vietnamese,Spanish,Chinese": 0.9235294117647059, "Vietnamese,English,Chinese": 0.9352941176470588, "Spanish,English,Chinese": 0.934453781512605 }, "4_combine": { "Vietnamese,Spanish,English,Chinese": 0.9193277310924369 } }, "AC3_2": 0.9539550082700835, "AC3_3": 0.9435039294077947, "AC3_4": 0.9357314017016041 }, "prompt_3": { "overall_acc": 0.9497899159663865, "language_acc": { "Vietnamese": 0.9529411764705882, "Spanish": 0.9478991596638655, "English": 0.9563025210084034, "Chinese": 0.9420168067226891 }, "consistency_score_2": 0.9507002801120449, "consistency_score_3": 0.9273109243697479, "consistency_score_4": 0.9100840336134454, "detailed_consistency_score": { "2_combine": { "Vietnamese,Spanish": 0.9521008403361344, "Vietnamese,English": 0.9630252100840336, "Vietnamese,Chinese": 0.9403361344537815, "Spanish,English": 0.9613445378151261, "Spanish,Chinese": 0.9352941176470588, "English,Chinese": 0.9521008403361344 }, "3_combine": { "Vietnamese,Spanish,English": 0.9394957983193277, "Vietnamese,Spanish,Chinese": 0.915126050420168, "Vietnamese,English,Chinese": 0.9285714285714286, "Spanish,English,Chinese": 0.9260504201680673 }, "4_combine": { "Vietnamese,Spanish,English,Chinese": 0.9100840336134454 } }, "AC3_2": 0.9502448799499753, "AC3_3": 0.9384158229104169, "AC3_4": 0.9295131403468117 }, "prompt_4": { "overall_acc": 0.9521008403361344, "language_acc": { "Vietnamese": 0.9470588235294117, "Spanish": 0.957983193277311, "English": 0.957983193277311, "Chinese": 0.9453781512605042 }, "consistency_score_2": 0.9511204481792718, "consistency_score_3": 0.928781512605042, "consistency_score_4": 0.9117647058823529, "detailed_consistency_score": { "2_combine": { "Vietnamese,Spanish": 0.9537815126050421, "Vietnamese,English": 0.9588235294117647, "Vietnamese,Chinese": 0.9361344537815126, "Spanish,English": 0.9621848739495799, "Spanish,Chinese": 0.9445378151260504, "English,Chinese": 0.9512605042016806 }, "3_combine": { "Vietnamese,Spanish,English": 0.9394957983193277, "Vietnamese,Spanish,Chinese": 0.9193277310924369, "Vietnamese,English,Chinese": 0.9252100840336135, "Spanish,English,Chinese": 0.9310924369747899 }, "4_combine": { "Vietnamese,Spanish,English,Chinese": 0.9117647058823529 } }, "AC3_2": 0.9516103916966612, "AC3_3": 0.9402966189885535, "AC3_4": 0.9314963135340684 }, "prompt_5": { "overall_acc": 0.9508403361344537, "language_acc": { "Vietnamese": 0.9529411764705882, "Spanish": 0.9487394957983193, "English": 0.9605042016806723, "Chinese": 0.9411764705882353 }, "consistency_score_2": 0.9481792717086835, "consistency_score_3": 0.9245798319327732, "consistency_score_4": 0.907563025210084, "detailed_consistency_score": { "2_combine": { "Vietnamese,Spanish": 0.9495798319327731, "Vietnamese,English": 0.9605042016806723, "Vietnamese,Chinese": 0.9369747899159664, "Spanish,English": 0.9571428571428572, "Spanish,Chinese": 0.9394957983193277, "English,Chinese": 0.9453781512605042 }, "3_combine": { "Vietnamese,Spanish,English": 0.9352941176470588, "Vietnamese,Spanish,Chinese": 0.915126050420168, "Vietnamese,English,Chinese": 0.9235294117647059, "Spanish,English,Chinese": 0.9243697478991597 }, "4_combine": { "Vietnamese,Spanish,English,Chinese": 0.907563025210084 } }, "AC3_2": 0.9495079394190248, "AC3_3": 0.9375262280955752, "AC3_4": 0.9286977734297558 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.8476190476190476, "language_acc": { "Chinese": 0.8533333333333334, "Indonesian": 0.84, "Spanish": 0.8533333333333334, "Vietnamese": 0.8333333333333334, "Malay": 0.82, "English": 0.8933333333333333, "Filipino": 0.84 }, "consistency_score_2": 0.8746031746031748, "consistency_score_3": 0.8192380952380951, "consistency_score_4": 0.7830476190476192, "consistency_score_5": 0.7546031746031745, "consistency_score_6": 0.7295238095238095, "consistency_score_7": 0.7066666666666667, "detailed_consistency_score": { "2_combine": { "Chinese,Indonesian": 0.8533333333333334, "Chinese,Spanish": 0.8733333333333333, "Chinese,Vietnamese": 0.8733333333333333, "Chinese,Malay": 0.8333333333333334, "Chinese,English": 0.9266666666666666, "Chinese,Filipino": 0.84, "Indonesian,Spanish": 0.9, "Indonesian,Vietnamese": 0.8933333333333333, "Indonesian,Malay": 0.9066666666666666, "Indonesian,English": 0.92, "Indonesian,Filipino": 0.8266666666666667, "Spanish,Vietnamese": 0.9, "Spanish,Malay": 0.8666666666666667, "Spanish,English": 0.9133333333333333, "Spanish,Filipino": 0.8266666666666667, "Vietnamese,Malay": 0.84, "Vietnamese,English": 0.9133333333333333, "Vietnamese,Filipino": 0.84, "Malay,English": 0.88, "Malay,Filipino": 0.8666666666666667, "English,Filipino": 0.8733333333333333 }, "3_combine": { "Chinese,Indonesian,Spanish": 0.8333333333333334, "Chinese,Indonesian,Vietnamese": 0.8133333333333334, "Chinese,Indonesian,Malay": 0.8066666666666666, "Chinese,Indonesian,English": 0.8533333333333334, "Chinese,Indonesian,Filipino": 0.7666666666666667, "Chinese,Spanish,Vietnamese": 0.8333333333333334, "Chinese,Spanish,Malay": 0.8066666666666666, "Chinese,Spanish,English": 0.8666666666666667, "Chinese,Spanish,Filipino": 0.7733333333333333, "Chinese,Vietnamese,Malay": 0.7866666666666666, "Chinese,Vietnamese,English": 0.86, "Chinese,Vietnamese,Filipino": 0.78, "Chinese,Malay,English": 0.8266666666666667, "Chinese,Malay,Filipino": 0.7733333333333333, "Chinese,English,Filipino": 0.8266666666666667, "Indonesian,Spanish,Vietnamese": 0.8533333333333334, "Indonesian,Spanish,Malay": 0.84, "Indonesian,Spanish,English": 0.8733333333333333, "Indonesian,Spanish,Filipino": 0.7933333333333333, "Indonesian,Vietnamese,Malay": 0.82, "Indonesian,Vietnamese,English": 0.8666666666666667, "Indonesian,Vietnamese,Filipino": 0.78, "Indonesian,Malay,English": 0.86, "Indonesian,Malay,Filipino": 0.8066666666666666, "Indonesian,English,Filipino": 0.8133333333333334, "Spanish,Vietnamese,Malay": 0.8133333333333334, "Spanish,Vietnamese,English": 0.8733333333333333, "Spanish,Vietnamese,Filipino": 0.7933333333333333, "Spanish,Malay,English": 0.84, "Spanish,Malay,Filipino": 0.7933333333333333, "Spanish,English,Filipino": 0.8133333333333334, "Vietnamese,Malay,English": 0.8266666666666667, "Vietnamese,Malay,Filipino": 0.78, "Vietnamese,English,Filipino": 0.8133333333333334, "Malay,English,Filipino": 0.8133333333333334 }, "4_combine": { "Chinese,Indonesian,Spanish,Vietnamese": 0.8, "Chinese,Indonesian,Spanish,Malay": 0.7933333333333333, "Chinese,Indonesian,Spanish,English": 0.8333333333333334, "Chinese,Indonesian,Spanish,Filipino": 0.7533333333333333, "Chinese,Indonesian,Vietnamese,Malay": 0.7733333333333333, "Chinese,Indonesian,Vietnamese,English": 0.8133333333333334, "Chinese,Indonesian,Vietnamese,Filipino": 0.7333333333333333, "Chinese,Indonesian,Malay,English": 0.8066666666666666, "Chinese,Indonesian,Malay,Filipino": 0.7533333333333333, "Chinese,Indonesian,English,Filipino": 0.7666666666666667, "Chinese,Spanish,Vietnamese,Malay": 0.7666666666666667, "Chinese,Spanish,Vietnamese,English": 0.8333333333333334, "Chinese,Spanish,Vietnamese,Filipino": 0.74, "Chinese,Spanish,Malay,English": 0.8, "Chinese,Spanish,Malay,Filipino": 0.7466666666666667, "Chinese,Spanish,English,Filipino": 0.7733333333333333, "Chinese,Vietnamese,Malay,English": 0.7866666666666666, "Chinese,Vietnamese,Malay,Filipino": 0.7333333333333333, "Chinese,Vietnamese,English,Filipino": 0.7666666666666667, "Chinese,Malay,English,Filipino": 0.7733333333333333, "Indonesian,Spanish,Vietnamese,Malay": 0.8, "Indonesian,Spanish,Vietnamese,English": 0.84, "Indonesian,Spanish,Vietnamese,Filipino": 0.76, "Indonesian,Spanish,Malay,English": 0.8333333333333334, "Indonesian,Spanish,Malay,Filipino": 0.78, "Indonesian,Spanish,English,Filipino": 0.7933333333333333, "Indonesian,Vietnamese,Malay,English": 0.8133333333333334, "Indonesian,Vietnamese,Malay,Filipino": 0.76, "Indonesian,Vietnamese,English,Filipino": 0.78, "Indonesian,Malay,English,Filipino": 0.7933333333333333, "Spanish,Vietnamese,Malay,English": 0.8066666666666666, "Spanish,Vietnamese,Malay,Filipino": 0.76, "Spanish,Vietnamese,English,Filipino": 0.78, "Spanish,Malay,English,Filipino": 0.7866666666666666, "Vietnamese,Malay,English,Filipino": 0.7733333333333333 }, "5_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay": 0.76, "Chinese,Indonesian,Spanish,Vietnamese,English": 0.8, "Chinese,Indonesian,Spanish,Vietnamese,Filipino": 0.72, "Chinese,Indonesian,Spanish,Malay,English": 0.7933333333333333, "Chinese,Indonesian,Spanish,Malay,Filipino": 0.74, "Chinese,Indonesian,Spanish,English,Filipino": 0.7533333333333333, "Chinese,Indonesian,Vietnamese,Malay,English": 0.7733333333333333, "Chinese,Indonesian,Vietnamese,Malay,Filipino": 0.72, "Chinese,Indonesian,Vietnamese,English,Filipino": 0.7333333333333333, "Chinese,Indonesian,Malay,English,Filipino": 0.7533333333333333, "Chinese,Spanish,Vietnamese,Malay,English": 0.7666666666666667, "Chinese,Spanish,Vietnamese,Malay,Filipino": 0.7133333333333334, "Chinese,Spanish,Vietnamese,English,Filipino": 0.74, "Chinese,Spanish,Malay,English,Filipino": 0.7466666666666667, "Chinese,Vietnamese,Malay,English,Filipino": 0.7333333333333333, "Indonesian,Spanish,Vietnamese,Malay,English": 0.8, "Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.7466666666666667, "Indonesian,Spanish,Vietnamese,English,Filipino": 0.76, "Indonesian,Spanish,Malay,English,Filipino": 0.78, "Indonesian,Vietnamese,Malay,English,Filipino": 0.76, "Spanish,Vietnamese,Malay,English,Filipino": 0.7533333333333333 }, "6_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English": 0.76, "Chinese,Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.7066666666666667, "Chinese,Indonesian,Spanish,Vietnamese,English,Filipino": 0.72, "Chinese,Indonesian,Spanish,Malay,English,Filipino": 0.74, "Chinese,Indonesian,Vietnamese,Malay,English,Filipino": 0.72, "Chinese,Spanish,Vietnamese,Malay,English,Filipino": 0.7133333333333334, "Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.7466666666666667 }, "7_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.7066666666666667 } }, "AC3_2": 0.8608997146746119, "AC3_3": 0.8331869554544192, "AC3_4": 0.8140548778292759, "AC3_5": 0.7984111258429666, "AC3_6": 0.784149988448095, "AC3_7": 0.7707516339373393 }, "prompt_2": { "overall_acc": 0.8542857142857143, "language_acc": { "Chinese": 0.8533333333333334, "Indonesian": 0.8533333333333334, "Spanish": 0.8533333333333334, "Vietnamese": 0.84, "Malay": 0.84, "English": 0.8933333333333333, "Filipino": 0.8466666666666667 }, "consistency_score_2": 0.8720634920634922, "consistency_score_3": 0.8161904761904762, "consistency_score_4": 0.7800000000000002, "consistency_score_5": 0.7523809523809523, "consistency_score_6": 0.7285714285714286, "consistency_score_7": 0.7066666666666667, "detailed_consistency_score": { "2_combine": { "Chinese,Indonesian": 0.88, "Chinese,Spanish": 0.8933333333333333, "Chinese,Vietnamese": 0.8733333333333333, "Chinese,Malay": 0.8666666666666667, "Chinese,English": 0.8933333333333333, "Chinese,Filipino": 0.8466666666666667, "Indonesian,Spanish": 0.8866666666666667, "Indonesian,Vietnamese": 0.8466666666666667, "Indonesian,Malay": 0.9, "Indonesian,English": 0.9066666666666666, "Indonesian,Filipino": 0.84, "Spanish,Vietnamese": 0.8533333333333334, "Spanish,Malay": 0.86, "Spanish,English": 0.9, "Spanish,Filipino": 0.8533333333333334, "Vietnamese,Malay": 0.8666666666666667, "Vietnamese,English": 0.88, "Vietnamese,Filipino": 0.84, "Malay,English": 0.8866666666666667, "Malay,Filipino": 0.8666666666666667, "English,Filipino": 0.8733333333333333 }, "3_combine": { "Chinese,Indonesian,Spanish": 0.84, "Chinese,Indonesian,Vietnamese": 0.8066666666666666, "Chinese,Indonesian,Malay": 0.8266666666666667, "Chinese,Indonesian,English": 0.8466666666666667, "Chinese,Indonesian,Filipino": 0.7933333333333333, "Chinese,Spanish,Vietnamese": 0.82, "Chinese,Spanish,Malay": 0.8133333333333334, "Chinese,Spanish,English": 0.8533333333333334, "Chinese,Spanish,Filipino": 0.8066666666666666, "Chinese,Vietnamese,Malay": 0.8066666666666666, "Chinese,Vietnamese,English": 0.8333333333333334, "Chinese,Vietnamese,Filipino": 0.7866666666666666, "Chinese,Malay,English": 0.8266666666666667, "Chinese,Malay,Filipino": 0.8066666666666666, "Chinese,English,Filipino": 0.8133333333333334, "Indonesian,Spanish,Vietnamese": 0.8066666666666666, "Indonesian,Spanish,Malay": 0.8333333333333334, "Indonesian,Spanish,English": 0.8533333333333334, "Indonesian,Spanish,Filipino": 0.8066666666666666, "Indonesian,Vietnamese,Malay": 0.8133333333333334, "Indonesian,Vietnamese,English": 0.8266666666666667, "Indonesian,Vietnamese,Filipino": 0.7733333333333333, "Indonesian,Malay,English": 0.86, "Indonesian,Malay,Filipino": 0.8133333333333334, "Indonesian,English,Filipino": 0.8133333333333334, "Spanish,Vietnamese,Malay": 0.8, "Spanish,Vietnamese,English": 0.82, "Spanish,Vietnamese,Filipino": 0.78, "Spanish,Malay,English": 0.8266666666666667, "Spanish,Malay,Filipino": 0.8066666666666666, "Spanish,English,Filipino": 0.82, "Vietnamese,Malay,English": 0.82, "Vietnamese,Malay,Filipino": 0.7933333333333333, "Vietnamese,English,Filipino": 0.8, "Malay,English,Filipino": 0.82 }, "4_combine": { "Chinese,Indonesian,Spanish,Vietnamese": 0.78, "Chinese,Indonesian,Spanish,Malay": 0.8, "Chinese,Indonesian,Spanish,English": 0.82, "Chinese,Indonesian,Spanish,Filipino": 0.78, "Chinese,Indonesian,Vietnamese,Malay": 0.7733333333333333, "Chinese,Indonesian,Vietnamese,English": 0.7933333333333333, "Chinese,Indonesian,Vietnamese,Filipino": 0.74, "Chinese,Indonesian,Malay,English": 0.8133333333333334, "Chinese,Indonesian,Malay,Filipino": 0.7733333333333333, "Chinese,Indonesian,English,Filipino": 0.78, "Chinese,Spanish,Vietnamese,Malay": 0.7666666666666667, "Chinese,Spanish,Vietnamese,English": 0.7933333333333333, "Chinese,Spanish,Vietnamese,Filipino": 0.7533333333333333, "Chinese,Spanish,Malay,English": 0.8, "Chinese,Spanish,Malay,Filipino": 0.7733333333333333, "Chinese,Spanish,English,Filipino": 0.7866666666666666, "Chinese,Vietnamese,Malay,English": 0.78, "Chinese,Vietnamese,Malay,Filipino": 0.7533333333333333, "Chinese,Vietnamese,English,Filipino": 0.7666666666666667, "Chinese,Malay,English,Filipino": 0.78, "Indonesian,Spanish,Vietnamese,Malay": 0.78, "Indonesian,Spanish,Vietnamese,English": 0.7933333333333333, "Indonesian,Spanish,Vietnamese,Filipino": 0.7533333333333333, "Indonesian,Spanish,Malay,English": 0.82, "Indonesian,Spanish,Malay,Filipino": 0.7866666666666666, "Indonesian,Spanish,English,Filipino": 0.7933333333333333, "Indonesian,Vietnamese,Malay,English": 0.8, "Indonesian,Vietnamese,Malay,Filipino": 0.76, "Indonesian,Vietnamese,English,Filipino": 0.76, "Indonesian,Malay,English,Filipino": 0.8, "Spanish,Vietnamese,Malay,English": 0.78, "Spanish,Vietnamese,Malay,Filipino": 0.7533333333333333, "Spanish,Vietnamese,English,Filipino": 0.7666666666666667, "Spanish,Malay,English,Filipino": 0.7866666666666666, "Vietnamese,Malay,English,Filipino": 0.76 }, "5_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay": 0.7533333333333333, "Chinese,Indonesian,Spanish,Vietnamese,English": 0.7666666666666667, "Chinese,Indonesian,Spanish,Vietnamese,Filipino": 0.7266666666666667, "Chinese,Indonesian,Spanish,Malay,English": 0.7933333333333333, "Chinese,Indonesian,Spanish,Malay,Filipino": 0.76, "Chinese,Indonesian,Spanish,English,Filipino": 0.7666666666666667, "Chinese,Indonesian,Vietnamese,Malay,English": 0.7666666666666667, "Chinese,Indonesian,Vietnamese,Malay,Filipino": 0.7266666666666667, "Chinese,Indonesian,Vietnamese,English,Filipino": 0.7333333333333333, "Chinese,Indonesian,Malay,English,Filipino": 0.7666666666666667, "Chinese,Spanish,Vietnamese,Malay,English": 0.7533333333333333, "Chinese,Spanish,Vietnamese,Malay,Filipino": 0.7266666666666667, "Chinese,Spanish,Vietnamese,English,Filipino": 0.74, "Chinese,Spanish,Malay,English,Filipino": 0.76, "Chinese,Vietnamese,Malay,English,Filipino": 0.7333333333333333, "Indonesian,Spanish,Vietnamese,Malay,English": 0.7733333333333333, "Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.74, "Indonesian,Spanish,Vietnamese,English,Filipino": 0.7466666666666667, "Indonesian,Spanish,Malay,English,Filipino": 0.78, "Indonesian,Vietnamese,Malay,English,Filipino": 0.7466666666666667, "Spanish,Vietnamese,Malay,English,Filipino": 0.74 }, "6_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English": 0.7466666666666667, "Chinese,Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.7133333333333334, "Chinese,Indonesian,Spanish,Vietnamese,English,Filipino": 0.72, "Chinese,Indonesian,Spanish,Malay,English,Filipino": 0.7533333333333333, "Chinese,Indonesian,Vietnamese,Malay,English,Filipino": 0.72, "Chinese,Spanish,Vietnamese,Malay,English,Filipino": 0.7133333333333334, "Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.7333333333333333 }, "7_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.7066666666666667 } }, "AC3_2": 0.8630830662033548, "AC3_3": 0.8348037139099542, "AC3_4": 0.8154545454046489, "AC3_5": 0.8001016173606508, "AC3_6": 0.786436307325251, "AC3_7": 0.7734960341176221 }, "prompt_3": { "overall_acc": 0.8561904761904762, "language_acc": { "Chinese": 0.86, "Indonesian": 0.8266666666666667, "Spanish": 0.8733333333333333, "Vietnamese": 0.8333333333333334, "Malay": 0.82, "English": 0.8933333333333333, "Filipino": 0.8866666666666667 }, "consistency_score_2": 0.8577777777777779, "consistency_score_3": 0.7963809523809525, "consistency_score_4": 0.7539047619047619, "consistency_score_5": 0.7200000000000001, "consistency_score_6": 0.6914285714285714, "consistency_score_7": 0.6666666666666666, "detailed_consistency_score": { "2_combine": { "Chinese,Indonesian": 0.8333333333333334, "Chinese,Spanish": 0.8666666666666667, "Chinese,Vietnamese": 0.8333333333333334, "Chinese,Malay": 0.84, "Chinese,English": 0.8733333333333333, "Chinese,Filipino": 0.88, "Indonesian,Spanish": 0.8466666666666667, "Indonesian,Vietnamese": 0.7866666666666666, "Indonesian,Malay": 0.8933333333333333, "Indonesian,English": 0.8466666666666667, "Indonesian,Filipino": 0.86, "Spanish,Vietnamese": 0.8733333333333333, "Spanish,Malay": 0.8466666666666667, "Spanish,English": 0.9, "Spanish,Filipino": 0.88, "Vietnamese,Malay": 0.8066666666666666, "Vietnamese,English": 0.8666666666666667, "Vietnamese,Filipino": 0.8333333333333334, "Malay,English": 0.8733333333333333, "Malay,Filipino": 0.86, "English,Filipino": 0.9133333333333333 }, "3_combine": { "Chinese,Indonesian,Spanish": 0.78, "Chinese,Indonesian,Vietnamese": 0.74, "Chinese,Indonesian,Malay": 0.8, "Chinese,Indonesian,English": 0.78, "Chinese,Indonesian,Filipino": 0.7933333333333333, "Chinese,Spanish,Vietnamese": 0.7933333333333333, "Chinese,Spanish,Malay": 0.7933333333333333, "Chinese,Spanish,English": 0.8333333333333334, "Chinese,Spanish,Filipino": 0.8266666666666667, "Chinese,Vietnamese,Malay": 0.76, "Chinese,Vietnamese,English": 0.8, "Chinese,Vietnamese,Filipino": 0.7866666666666666, "Chinese,Malay,English": 0.8, "Chinese,Malay,Filipino": 0.8, "Chinese,English,Filipino": 0.84, "Indonesian,Spanish,Vietnamese": 0.7666666666666667, "Indonesian,Spanish,Malay": 0.8066666666666666, "Indonesian,Spanish,English": 0.8066666666666666, "Indonesian,Spanish,Filipino": 0.8066666666666666, "Indonesian,Vietnamese,Malay": 0.7533333333333333, "Indonesian,Vietnamese,English": 0.7666666666666667, "Indonesian,Vietnamese,Filipino": 0.7533333333333333, "Indonesian,Malay,English": 0.8133333333333334, "Indonesian,Malay,Filipino": 0.8133333333333334, "Indonesian,English,Filipino": 0.8133333333333334, "Spanish,Vietnamese,Malay": 0.7733333333333333, "Spanish,Vietnamese,English": 0.8266666666666667, "Spanish,Vietnamese,Filipino": 0.8, "Spanish,Malay,English": 0.8133333333333334, "Spanish,Malay,Filipino": 0.8, "Spanish,English,Filipino": 0.8533333333333334, "Vietnamese,Malay,English": 0.78, "Vietnamese,Malay,Filipino": 0.76, "Vietnamese,English,Filipino": 0.8133333333333334, "Malay,English,Filipino": 0.8266666666666667 }, "4_combine": { "Chinese,Indonesian,Spanish,Vietnamese": 0.72, "Chinese,Indonesian,Spanish,Malay": 0.76, "Chinese,Indonesian,Spanish,English": 0.7533333333333333, "Chinese,Indonesian,Spanish,Filipino": 0.76, "Chinese,Indonesian,Vietnamese,Malay": 0.72, "Chinese,Indonesian,Vietnamese,English": 0.72, "Chinese,Indonesian,Vietnamese,Filipino": 0.72, "Chinese,Indonesian,Malay,English": 0.76, "Chinese,Indonesian,Malay,Filipino": 0.7733333333333333, "Chinese,Indonesian,English,Filipino": 0.7666666666666667, "Chinese,Spanish,Vietnamese,Malay": 0.7333333333333333, "Chinese,Spanish,Vietnamese,English": 0.7666666666666667, "Chinese,Spanish,Vietnamese,Filipino": 0.76, "Chinese,Spanish,Malay,English": 0.7666666666666667, "Chinese,Spanish,Malay,Filipino": 0.7666666666666667, "Chinese,Spanish,English,Filipino": 0.8066666666666666, "Chinese,Vietnamese,Malay,English": 0.74, "Chinese,Vietnamese,Malay,Filipino": 0.7266666666666667, "Chinese,Vietnamese,English,Filipino": 0.7733333333333333, "Chinese,Malay,English,Filipino": 0.7733333333333333, "Indonesian,Spanish,Vietnamese,Malay": 0.7333333333333333, "Indonesian,Spanish,Vietnamese,English": 0.7466666666666667, "Indonesian,Spanish,Vietnamese,Filipino": 0.7333333333333333, "Indonesian,Spanish,Malay,English": 0.7733333333333333, "Indonesian,Spanish,Malay,Filipino": 0.7733333333333333, "Indonesian,Spanish,English,Filipino": 0.78, "Indonesian,Vietnamese,Malay,English": 0.7333333333333333, "Indonesian,Vietnamese,Malay,Filipino": 0.7266666666666667, "Indonesian,Vietnamese,English,Filipino": 0.74, "Indonesian,Malay,English,Filipino": 0.7866666666666666, "Spanish,Vietnamese,Malay,English": 0.7533333333333333, "Spanish,Vietnamese,Malay,Filipino": 0.7333333333333333, "Spanish,Vietnamese,English,Filipino": 0.7866666666666666, "Spanish,Malay,English,Filipino": 0.78, "Vietnamese,Malay,English,Filipino": 0.74 }, "5_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay": 0.7, "Chinese,Indonesian,Spanish,Vietnamese,English": 0.7, "Chinese,Indonesian,Spanish,Vietnamese,Filipino": 0.7, "Chinese,Indonesian,Spanish,Malay,English": 0.7333333333333333, "Chinese,Indonesian,Spanish,Malay,Filipino": 0.74, "Chinese,Indonesian,Spanish,English,Filipino": 0.74, "Chinese,Indonesian,Vietnamese,Malay,English": 0.7, "Chinese,Indonesian,Vietnamese,Malay,Filipino": 0.7, "Chinese,Indonesian,Vietnamese,English,Filipino": 0.7066666666666667, "Chinese,Indonesian,Malay,English,Filipino": 0.7466666666666667, "Chinese,Spanish,Vietnamese,Malay,English": 0.7133333333333334, "Chinese,Spanish,Vietnamese,Malay,Filipino": 0.7066666666666667, "Chinese,Spanish,Vietnamese,English,Filipino": 0.7466666666666667, "Chinese,Spanish,Malay,English,Filipino": 0.7466666666666667, "Chinese,Vietnamese,Malay,English,Filipino": 0.7133333333333334, "Indonesian,Spanish,Vietnamese,Malay,English": 0.7133333333333334, "Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.7066666666666667, "Indonesian,Spanish,Vietnamese,English,Filipino": 0.72, "Indonesian,Spanish,Malay,English,Filipino": 0.7533333333333333, "Indonesian,Vietnamese,Malay,English,Filipino": 0.7133333333333334, "Spanish,Vietnamese,Malay,English,Filipino": 0.72 }, "6_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English": 0.68, "Chinese,Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.68, "Chinese,Indonesian,Spanish,Vietnamese,English,Filipino": 0.6866666666666666, "Chinese,Indonesian,Spanish,Malay,English,Filipino": 0.72, "Chinese,Indonesian,Vietnamese,Malay,English,Filipino": 0.6866666666666666, "Chinese,Spanish,Vietnamese,Malay,English,Filipino": 0.6933333333333334, "Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.6933333333333334 }, "7_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.6666666666666666 } }, "AC3_2": 0.8569833919361705, "AC3_3": 0.8252034072696555, "AC3_4": 0.8017986287830197, "AC3_5": 0.7822114803129111, "AC3_6": 0.7650391208296875, "AC3_7": 0.7496351886103534 }, "prompt_4": { "overall_acc": 0.8447619047619048, "language_acc": { "Chinese": 0.8266666666666667, "Indonesian": 0.8333333333333334, "Spanish": 0.8533333333333334, "Vietnamese": 0.82, "Malay": 0.8333333333333334, "English": 0.92, "Filipino": 0.8266666666666667 }, "consistency_score_2": 0.8542857142857142, "consistency_score_3": 0.7904761904761904, "consistency_score_4": 0.7443809523809525, "consistency_score_5": 0.7057142857142857, "consistency_score_6": 0.6714285714285715, "consistency_score_7": 0.64, "detailed_consistency_score": { "2_combine": { "Chinese,Indonesian": 0.82, "Chinese,Spanish": 0.8666666666666667, "Chinese,Vietnamese": 0.8533333333333334, "Chinese,Malay": 0.8266666666666667, "Chinese,English": 0.8733333333333333, "Chinese,Filipino": 0.8133333333333334, "Indonesian,Spanish": 0.86, "Indonesian,Vietnamese": 0.84, "Indonesian,Malay": 0.8666666666666667, "Indonesian,English": 0.8933333333333333, "Indonesian,Filipino": 0.8333333333333334, "Spanish,Vietnamese": 0.8866666666666667, "Spanish,Malay": 0.8866666666666667, "Spanish,English": 0.8933333333333333, "Spanish,Filipino": 0.8466666666666667, "Vietnamese,Malay": 0.8333333333333334, "Vietnamese,English": 0.8866666666666667, "Vietnamese,Filipino": 0.84, "Malay,English": 0.8533333333333334, "Malay,Filipino": 0.8066666666666666, "English,Filipino": 0.86 }, "3_combine": { "Chinese,Indonesian,Spanish": 0.78, "Chinese,Indonesian,Vietnamese": 0.7666666666666667, "Chinese,Indonesian,Malay": 0.7666666666666667, "Chinese,Indonesian,English": 0.8066666666666666, "Chinese,Indonesian,Filipino": 0.7466666666666667, "Chinese,Spanish,Vietnamese": 0.8066666666666666, "Chinese,Spanish,Malay": 0.8, "Chinese,Spanish,English": 0.82, "Chinese,Spanish,Filipino": 0.7666666666666667, "Chinese,Vietnamese,Malay": 0.76, "Chinese,Vietnamese,English": 0.8066666666666666, "Chinese,Vietnamese,Filipino": 0.76, "Chinese,Malay,English": 0.7866666666666666, "Chinese,Malay,Filipino": 0.7333333333333333, "Chinese,English,Filipino": 0.7866666666666666, "Indonesian,Spanish,Vietnamese": 0.8066666666666666, "Indonesian,Spanish,Malay": 0.82, "Indonesian,Spanish,English": 0.8333333333333334, "Indonesian,Spanish,Filipino": 0.78, "Indonesian,Vietnamese,Malay": 0.7866666666666666, "Indonesian,Vietnamese,English": 0.8266666666666667, "Indonesian,Vietnamese,Filipino": 0.7666666666666667, "Indonesian,Malay,English": 0.82, "Indonesian,Malay,Filipino": 0.7666666666666667, "Indonesian,English,Filipino": 0.8, "Spanish,Vietnamese,Malay": 0.8066666666666666, "Spanish,Vietnamese,English": 0.84, "Spanish,Vietnamese,Filipino": 0.7933333333333333, "Spanish,Malay,English": 0.82, "Spanish,Malay,Filipino": 0.78, "Spanish,English,Filipino": 0.8066666666666666, "Vietnamese,Malay,English": 0.7933333333333333, "Vietnamese,Malay,Filipino": 0.7533333333333333, "Vietnamese,English,Filipino": 0.8066666666666666, "Malay,English,Filipino": 0.7666666666666667 }, "4_combine": { "Chinese,Indonesian,Spanish,Vietnamese": 0.74, "Chinese,Indonesian,Spanish,Malay": 0.7466666666666667, "Chinese,Indonesian,Spanish,English": 0.7733333333333333, "Chinese,Indonesian,Spanish,Filipino": 0.7133333333333334, "Chinese,Indonesian,Vietnamese,Malay": 0.7266666666666667, "Chinese,Indonesian,Vietnamese,English": 0.76, "Chinese,Indonesian,Vietnamese,Filipino": 0.7, "Chinese,Indonesian,Malay,English": 0.76, "Chinese,Indonesian,Malay,Filipino": 0.7066666666666667, "Chinese,Indonesian,English,Filipino": 0.74, "Chinese,Spanish,Vietnamese,Malay": 0.74, "Chinese,Spanish,Vietnamese,English": 0.7733333333333333, "Chinese,Spanish,Vietnamese,Filipino": 0.7266666666666667, "Chinese,Spanish,Malay,English": 0.76, "Chinese,Spanish,Malay,Filipino": 0.7133333333333334, "Chinese,Spanish,English,Filipino": 0.7466666666666667, "Chinese,Vietnamese,Malay,English": 0.7333333333333333, "Chinese,Vietnamese,Malay,Filipino": 0.6933333333333334, "Chinese,Vietnamese,English,Filipino": 0.74, "Chinese,Malay,English,Filipino": 0.7133333333333334, "Indonesian,Spanish,Vietnamese,Malay": 0.7666666666666667, "Indonesian,Spanish,Vietnamese,English": 0.7933333333333333, "Indonesian,Spanish,Vietnamese,Filipino": 0.7333333333333333, "Indonesian,Spanish,Malay,English": 0.8, "Indonesian,Spanish,Malay,Filipino": 0.7466666666666667, "Indonesian,Spanish,English,Filipino": 0.76, "Indonesian,Vietnamese,Malay,English": 0.78, "Indonesian,Vietnamese,Malay,Filipino": 0.72, "Indonesian,Vietnamese,English,Filipino": 0.7533333333333333, "Indonesian,Malay,English,Filipino": 0.7533333333333333, "Spanish,Vietnamese,Malay,English": 0.7733333333333333, "Spanish,Vietnamese,Malay,Filipino": 0.7333333333333333, "Spanish,Vietnamese,English,Filipino": 0.76, "Spanish,Malay,English,Filipino": 0.7466666666666667, "Vietnamese,Malay,English,Filipino": 0.7266666666666667 }, "5_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay": 0.7066666666666667, "Chinese,Indonesian,Spanish,Vietnamese,English": 0.7333333333333333, "Chinese,Indonesian,Spanish,Vietnamese,Filipino": 0.6733333333333333, "Chinese,Indonesian,Spanish,Malay,English": 0.74, "Chinese,Indonesian,Spanish,Malay,Filipino": 0.6866666666666666, "Chinese,Indonesian,Spanish,English,Filipino": 0.7066666666666667, "Chinese,Indonesian,Vietnamese,Malay,English": 0.72, "Chinese,Indonesian,Vietnamese,Malay,Filipino": 0.6666666666666666, "Chinese,Indonesian,Vietnamese,English,Filipino": 0.6933333333333334, "Chinese,Indonesian,Malay,English,Filipino": 0.7, "Chinese,Spanish,Vietnamese,Malay,English": 0.7133333333333334, "Chinese,Spanish,Vietnamese,Malay,Filipino": 0.6733333333333333, "Chinese,Spanish,Vietnamese,English,Filipino": 0.7066666666666667, "Chinese,Spanish,Malay,English,Filipino": 0.6933333333333334, "Chinese,Vietnamese,Malay,English,Filipino": 0.6733333333333333, "Indonesian,Spanish,Vietnamese,Malay,English": 0.76, "Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.7, "Indonesian,Spanish,Vietnamese,English,Filipino": 0.72, "Indonesian,Spanish,Malay,English,Filipino": 0.7333333333333333, "Indonesian,Vietnamese,Malay,English,Filipino": 0.7133333333333334, "Spanish,Vietnamese,Malay,English,Filipino": 0.7066666666666667 }, "6_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English": 0.7, "Chinese,Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.6466666666666666, "Chinese,Indonesian,Spanish,Vietnamese,English,Filipino": 0.6666666666666666, "Chinese,Indonesian,Spanish,Malay,English,Filipino": 0.68, "Chinese,Indonesian,Vietnamese,Malay,English,Filipino": 0.66, "Chinese,Spanish,Vietnamese,Malay,English,Filipino": 0.6533333333333333, "Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.6933333333333334 }, "7_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.64 } }, "AC3_2": 0.8494971171825447, "AC3_3": 0.8167179742684954, "AC3_4": 0.7914010604343237, "AC3_5": 0.7690031589535612, "AC3_6": 0.7481873653490742, "AC3_7": 0.7282617061728882 }, "prompt_5": { "overall_acc": 0.8514285714285713, "language_acc": { "Chinese": 0.8466666666666667, "Indonesian": 0.84, "Spanish": 0.8466666666666667, "Vietnamese": 0.82, "Malay": 0.8533333333333334, "English": 0.8866666666666667, "Filipino": 0.8666666666666667 }, "consistency_score_2": 0.8536507936507934, "consistency_score_3": 0.7893333333333331, "consistency_score_4": 0.7460952380952381, "consistency_score_5": 0.7120634920634921, "consistency_score_6": 0.6838095238095239, "consistency_score_7": 0.66, "detailed_consistency_score": { "2_combine": { "Chinese,Indonesian": 0.8133333333333334, "Chinese,Spanish": 0.84, "Chinese,Vietnamese": 0.8666666666666667, "Chinese,Malay": 0.84, "Chinese,English": 0.8666666666666667, "Chinese,Filipino": 0.8666666666666667, "Indonesian,Spanish": 0.8333333333333334, "Indonesian,Vietnamese": 0.8333333333333334, "Indonesian,Malay": 0.8666666666666667, "Indonesian,English": 0.8466666666666667, "Indonesian,Filipino": 0.8333333333333334, "Spanish,Vietnamese": 0.84, "Spanish,Malay": 0.8466666666666667, "Spanish,English": 0.8866666666666667, "Spanish,Filipino": 0.84, "Vietnamese,Malay": 0.86, "Vietnamese,English": 0.8733333333333333, "Vietnamese,Filipino": 0.88, "Malay,English": 0.9, "Malay,Filipino": 0.8333333333333334, "English,Filipino": 0.86 }, "3_combine": { "Chinese,Indonesian,Spanish": 0.7533333333333333, "Chinese,Indonesian,Vietnamese": 0.7666666666666667, "Chinese,Indonesian,Malay": 0.7666666666666667, "Chinese,Indonesian,English": 0.7733333333333333, "Chinese,Indonesian,Filipino": 0.76, "Chinese,Spanish,Vietnamese": 0.7866666666666666, "Chinese,Spanish,Malay": 0.7666666666666667, "Chinese,Spanish,English": 0.8, "Chinese,Spanish,Filipino": 0.78, "Chinese,Vietnamese,Malay": 0.7933333333333333, "Chinese,Vietnamese,English": 0.8133333333333334, "Chinese,Vietnamese,Filipino": 0.8133333333333334, "Chinese,Malay,English": 0.8133333333333334, "Chinese,Malay,Filipino": 0.78, "Chinese,English,Filipino": 0.8, "Indonesian,Spanish,Vietnamese": 0.7733333333333333, "Indonesian,Spanish,Malay": 0.78, "Indonesian,Spanish,English": 0.7933333333333333, "Indonesian,Spanish,Filipino": 0.76, "Indonesian,Vietnamese,Malay": 0.7933333333333333, "Indonesian,Vietnamese,English": 0.7866666666666666, "Indonesian,Vietnamese,Filipino": 0.7866666666666666, "Indonesian,Malay,English": 0.8133333333333334, "Indonesian,Malay,Filipino": 0.7666666666666667, "Indonesian,English,Filipino": 0.7733333333333333, "Spanish,Vietnamese,Malay": 0.7866666666666666, "Spanish,Vietnamese,English": 0.8066666666666666, "Spanish,Vietnamese,Filipino": 0.7933333333333333, "Spanish,Malay,English": 0.82, "Spanish,Malay,Filipino": 0.7666666666666667, "Spanish,English,Filipino": 0.8, "Vietnamese,Malay,English": 0.8266666666666667, "Vietnamese,Malay,Filipino": 0.8066666666666666, "Vietnamese,English,Filipino": 0.82, "Malay,English,Filipino": 0.8066666666666666 }, "4_combine": { "Chinese,Indonesian,Spanish,Vietnamese": 0.7266666666666667, "Chinese,Indonesian,Spanish,Malay": 0.7266666666666667, "Chinese,Indonesian,Spanish,English": 0.7333333333333333, "Chinese,Indonesian,Spanish,Filipino": 0.7133333333333334, "Chinese,Indonesian,Vietnamese,Malay": 0.74, "Chinese,Indonesian,Vietnamese,English": 0.74, "Chinese,Indonesian,Vietnamese,Filipino": 0.7333333333333333, "Chinese,Indonesian,Malay,English": 0.7466666666666667, "Chinese,Indonesian,Malay,Filipino": 0.72, "Chinese,Indonesian,English,Filipino": 0.7266666666666667, "Chinese,Spanish,Vietnamese,Malay": 0.74, "Chinese,Spanish,Vietnamese,English": 0.76, "Chinese,Spanish,Vietnamese,Filipino": 0.7466666666666667, "Chinese,Spanish,Malay,English": 0.7533333333333333, "Chinese,Spanish,Malay,Filipino": 0.72, "Chinese,Spanish,English,Filipino": 0.7466666666666667, "Chinese,Vietnamese,Malay,English": 0.7733333333333333, "Chinese,Vietnamese,Malay,Filipino": 0.76, "Chinese,Vietnamese,English,Filipino": 0.7733333333333333, "Chinese,Malay,English,Filipino": 0.7533333333333333, "Indonesian,Spanish,Vietnamese,Malay": 0.7466666666666667, "Indonesian,Spanish,Vietnamese,English": 0.7466666666666667, "Indonesian,Spanish,Vietnamese,Filipino": 0.7333333333333333, "Indonesian,Spanish,Malay,English": 0.7666666666666667, "Indonesian,Spanish,Malay,Filipino": 0.7266666666666667, "Indonesian,Spanish,English,Filipino": 0.7333333333333333, "Indonesian,Vietnamese,Malay,English": 0.7666666666666667, "Indonesian,Vietnamese,Malay,Filipino": 0.7466666666666667, "Indonesian,Vietnamese,English,Filipino": 0.7466666666666667, "Indonesian,Malay,English,Filipino": 0.7466666666666667, "Spanish,Vietnamese,Malay,English": 0.7733333333333333, "Spanish,Vietnamese,Malay,Filipino": 0.7466666666666667, "Spanish,Vietnamese,English,Filipino": 0.76, "Spanish,Malay,English,Filipino": 0.7533333333333333, "Vietnamese,Malay,English,Filipino": 0.7866666666666666 }, "5_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay": 0.7066666666666667, "Chinese,Indonesian,Spanish,Vietnamese,English": 0.7066666666666667, "Chinese,Indonesian,Spanish,Vietnamese,Filipino": 0.6933333333333334, "Chinese,Indonesian,Spanish,Malay,English": 0.7133333333333334, "Chinese,Indonesian,Spanish,Malay,Filipino": 0.6866666666666666, "Chinese,Indonesian,Spanish,English,Filipino": 0.6933333333333334, "Chinese,Indonesian,Vietnamese,Malay,English": 0.72, "Chinese,Indonesian,Vietnamese,Malay,Filipino": 0.7066666666666667, "Chinese,Indonesian,Vietnamese,English,Filipino": 0.7066666666666667, "Chinese,Indonesian,Malay,English,Filipino": 0.7, "Chinese,Spanish,Vietnamese,Malay,English": 0.7266666666666667, "Chinese,Spanish,Vietnamese,Malay,Filipino": 0.7066666666666667, "Chinese,Spanish,Vietnamese,English,Filipino": 0.72, "Chinese,Spanish,Malay,English,Filipino": 0.7066666666666667, "Chinese,Vietnamese,Malay,English,Filipino": 0.74, "Indonesian,Spanish,Vietnamese,Malay,English": 0.7333333333333333, "Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.7066666666666667, "Indonesian,Spanish,Vietnamese,English,Filipino": 0.7066666666666667, "Indonesian,Spanish,Malay,English,Filipino": 0.7133333333333334, "Indonesian,Vietnamese,Malay,English,Filipino": 0.7266666666666667, "Spanish,Vietnamese,Malay,English,Filipino": 0.7333333333333333 }, "6_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English": 0.6933333333333334, "Chinese,Indonesian,Spanish,Vietnamese,Malay,Filipino": 0.6733333333333333, "Chinese,Indonesian,Spanish,Vietnamese,English,Filipino": 0.6733333333333333, "Chinese,Indonesian,Spanish,Malay,English,Filipino": 0.6733333333333333, "Chinese,Indonesian,Vietnamese,Malay,English,Filipino": 0.6866666666666666, "Chinese,Spanish,Vietnamese,Malay,English,Filipino": 0.6933333333333334, "Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.6933333333333334 }, "7_combine": { "Chinese,Indonesian,Spanish,Vietnamese,Malay,English,Filipino": 0.66 } }, "AC3_2": 0.8525382343835983, "AC3_3": 0.8192059437624698, "AC3_4": 0.7952893082334447, "AC3_5": 0.7755347352658433, "AC3_6": 0.7584686280966435, "AC3_7": 0.7435916823704618 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.6777597402597403, "language_acc": { "Vietnamese": 0.6477272727272727, "Indonesian": 0.6875, "Spanish": 0.6761363636363636, "Malay": 0.6590909090909091, "Filipino": 0.6534090909090909, "English": 0.7272727272727273, "Chinese": 0.6931818181818182 }, "consistency_score_2": 0.7732683982683982, "consistency_score_3": 0.6728896103896106, "consistency_score_4": 0.6136363636363636, "consistency_score_5": 0.5741341991341992, "consistency_score_6": 0.5454545454545454, "consistency_score_7": 0.5227272727272727, "detailed_consistency_score": { "2_combine": { "Vietnamese,Indonesian": 0.7613636363636364, "Vietnamese,Spanish": 0.7727272727272727, "Vietnamese,Malay": 0.7443181818181818, "Vietnamese,Filipino": 0.7613636363636364, "Vietnamese,English": 0.8181818181818182, "Vietnamese,Chinese": 0.7215909090909091, "Indonesian,Spanish": 0.8011363636363636, "Indonesian,Malay": 0.8125, "Indonesian,Filipino": 0.75, "Indonesian,English": 0.7840909090909091, "Indonesian,Chinese": 0.7670454545454546, "Spanish,Malay": 0.7897727272727273, "Spanish,Filipino": 0.8068181818181818, "Spanish,English": 0.8409090909090909, "Spanish,Chinese": 0.7556818181818182, "Malay,Filipino": 0.7613636363636364, "Malay,English": 0.7840909090909091, "Malay,Chinese": 0.7329545454545454, "Filipino,English": 0.8238636363636364, "Filipino,Chinese": 0.7102272727272727, "English,Chinese": 0.7386363636363636 }, "3_combine": { "Vietnamese,Indonesian,Spanish": 0.6818181818181818, "Vietnamese,Indonesian,Malay": 0.6761363636363636, "Vietnamese,Indonesian,Filipino": 0.6534090909090909, "Vietnamese,Indonesian,English": 0.6931818181818182, "Vietnamese,Indonesian,Chinese": 0.6363636363636364, "Vietnamese,Spanish,Malay": 0.6704545454545454, "Vietnamese,Spanish,Filipino": 0.6818181818181818, "Vietnamese,Spanish,English": 0.7329545454545454, "Vietnamese,Spanish,Chinese": 0.6363636363636364, "Vietnamese,Malay,Filipino": 0.6534090909090909, "Vietnamese,Malay,English": 0.6875, "Vietnamese,Malay,Chinese": 0.6136363636363636, "Vietnamese,Filipino,English": 0.7102272727272727, "Vietnamese,Filipino,Chinese": 0.6079545454545454, "Vietnamese,English,Chinese": 0.6477272727272727, "Indonesian,Spanish,Malay": 0.7102272727272727, "Indonesian,Spanish,Filipino": 0.6988636363636364, "Indonesian,Spanish,English": 0.7272727272727273, "Indonesian,Spanish,Chinese": 0.6761363636363636, "Indonesian,Malay,Filipino": 0.6875, "Indonesian,Malay,English": 0.7045454545454546, "Indonesian,Malay,Chinese": 0.6704545454545454, "Indonesian,Filipino,English": 0.6931818181818182, "Indonesian,Filipino,Chinese": 0.625, "Indonesian,English,Chinese": 0.6590909090909091, "Spanish,Malay,Filipino": 0.6931818181818182, "Spanish,Malay,English": 0.7102272727272727, "Spanish,Malay,Chinese": 0.6534090909090909, "Spanish,Filipino,English": 0.7443181818181818, "Spanish,Filipino,Chinese": 0.6477272727272727, "Spanish,English,Chinese": 0.6704545454545454, "Malay,Filipino,English": 0.6988636363636364, "Malay,Filipino,Chinese": 0.6193181818181818, "Malay,English,Chinese": 0.6363636363636364, "Filipino,English,Chinese": 0.6420454545454546 }, "4_combine": { "Vietnamese,Indonesian,Spanish,Malay": 0.6363636363636364, "Vietnamese,Indonesian,Spanish,Filipino": 0.625, "Vietnamese,Indonesian,Spanish,English": 0.6534090909090909, "Vietnamese,Indonesian,Spanish,Chinese": 0.5965909090909091, "Vietnamese,Indonesian,Malay,Filipino": 0.6193181818181818, "Vietnamese,Indonesian,Malay,English": 0.6306818181818182, "Vietnamese,Indonesian,Malay,Chinese": 0.5795454545454546, "Vietnamese,Indonesian,Filipino,English": 0.625, "Vietnamese,Indonesian,Filipino,Chinese": 0.5681818181818182, "Vietnamese,Indonesian,English,Chinese": 0.5965909090909091, "Vietnamese,Spanish,Malay,Filipino": 0.6136363636363636, "Vietnamese,Spanish,Malay,English": 0.6420454545454546, "Vietnamese,Spanish,Malay,Chinese": 0.5795454545454546, "Vietnamese,Spanish,Filipino,English": 0.6647727272727273, "Vietnamese,Spanish,Filipino,Chinese": 0.5795454545454546, "Vietnamese,Spanish,English,Chinese": 0.6079545454545454, "Vietnamese,Malay,Filipino,English": 0.625, "Vietnamese,Malay,Filipino,Chinese": 0.5511363636363636, "Vietnamese,Malay,English,Chinese": 0.5738636363636364, "Vietnamese,Filipino,English,Chinese": 0.5909090909090909, "Indonesian,Spanish,Malay,Filipino": 0.6477272727272727, "Indonesian,Spanish,Malay,English": 0.6704545454545454, "Indonesian,Spanish,Malay,Chinese": 0.6079545454545454, "Indonesian,Spanish,Filipino,English": 0.6647727272727273, "Indonesian,Spanish,Filipino,Chinese": 0.6079545454545454, "Indonesian,Spanish,English,Chinese": 0.625, "Indonesian,Malay,Filipino,English": 0.6477272727272727, "Indonesian,Malay,Filipino,Chinese": 0.5852272727272727, "Indonesian,Malay,English,Chinese": 0.6079545454545454, "Indonesian,Filipino,English,Chinese": 0.6022727272727273, "Spanish,Malay,Filipino,English": 0.6534090909090909, "Spanish,Malay,Filipino,Chinese": 0.5909090909090909, "Spanish,Malay,English,Chinese": 0.6079545454545454, "Spanish,Filipino,English,Chinese": 0.6079545454545454, "Malay,Filipino,English,Chinese": 0.5909090909090909 }, "5_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino": 0.5965909090909091, "Vietnamese,Indonesian,Spanish,Malay,English": 0.6136363636363636, "Vietnamese,Indonesian,Spanish,Malay,Chinese": 0.5568181818181818, "Vietnamese,Indonesian,Spanish,Filipino,English": 0.6079545454545454, "Vietnamese,Indonesian,Spanish,Filipino,Chinese": 0.5625, "Vietnamese,Indonesian,Spanish,English,Chinese": 0.5738636363636364, "Vietnamese,Indonesian,Malay,Filipino,English": 0.5909090909090909, "Vietnamese,Indonesian,Malay,Filipino,Chinese": 0.5397727272727273, "Vietnamese,Indonesian,Malay,English,Chinese": 0.5511363636363636, "Vietnamese,Indonesian,Filipino,English,Chinese": 0.5568181818181818, "Vietnamese,Spanish,Malay,Filipino,English": 0.5965909090909091, "Vietnamese,Spanish,Malay,Filipino,Chinese": 0.5397727272727273, "Vietnamese,Spanish,Malay,English,Chinese": 0.5568181818181818, "Vietnamese,Spanish,Filipino,English,Chinese": 0.5681818181818182, "Vietnamese,Malay,Filipino,English,Chinese": 0.5397727272727273, "Indonesian,Spanish,Malay,Filipino,English": 0.625, "Indonesian,Spanish,Malay,Filipino,Chinese": 0.5681818181818182, "Indonesian,Spanish,Malay,English,Chinese": 0.5852272727272727, "Indonesian,Spanish,Filipino,English,Chinese": 0.5852272727272727, "Indonesian,Malay,Filipino,English,Chinese": 0.5738636363636364, "Spanish,Malay,Filipino,English,Chinese": 0.5681818181818182 }, "6_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English": 0.5795454545454546, "Vietnamese,Indonesian,Spanish,Malay,Filipino,Chinese": 0.5340909090909091, "Vietnamese,Indonesian,Spanish,Malay,English,Chinese": 0.5397727272727273, "Vietnamese,Indonesian,Spanish,Filipino,English,Chinese": 0.5511363636363636, "Vietnamese,Indonesian,Malay,Filipino,English,Chinese": 0.5284090909090909, "Vietnamese,Spanish,Malay,Filipino,English,Chinese": 0.5284090909090909, "Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.5568181818181818 }, "7_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.5227272727272727 } }, "AC3_2": 0.7223708139208814, "AC3_3": 0.6753158949924584, "AC3_4": 0.6441060510329186, "AC3_5": 0.6216581667241351, "AC3_6": 0.6044519514496499, "AC3_7": 0.5902329583379504 }, "prompt_2": { "overall_acc": 0.6525974025974026, "language_acc": { "Vietnamese": 0.6079545454545454, "Indonesian": 0.6761363636363636, "Spanish": 0.6590909090909091, "Malay": 0.6079545454545454, "Filipino": 0.6306818181818182, "English": 0.7159090909090909, "Chinese": 0.6704545454545454 }, "consistency_score_2": 0.7337662337662337, "consistency_score_3": 0.6181818181818182, "consistency_score_4": 0.550974025974026, "consistency_score_5": 0.5064935064935066, "consistency_score_6": 0.4740259740259741, "consistency_score_7": 0.44886363636363635, "detailed_consistency_score": { "2_combine": { "Vietnamese,Indonesian": 0.7045454545454546, "Vietnamese,Spanish": 0.75, "Vietnamese,Malay": 0.7272727272727273, "Vietnamese,Filipino": 0.6931818181818182, "Vietnamese,English": 0.7272727272727273, "Vietnamese,Chinese": 0.6761363636363636, "Indonesian,Spanish": 0.7840909090909091, "Indonesian,Malay": 0.7727272727272727, "Indonesian,Filipino": 0.75, "Indonesian,English": 0.7670454545454546, "Indonesian,Chinese": 0.7159090909090909, "Spanish,Malay": 0.7443181818181818, "Spanish,Filipino": 0.7670454545454546, "Spanish,English": 0.7840909090909091, "Spanish,Chinese": 0.7215909090909091, "Malay,Filipino": 0.7386363636363636, "Malay,English": 0.7670454545454546, "Malay,Chinese": 0.6477272727272727, "Filipino,English": 0.7613636363636364, "Filipino,Chinese": 0.6704545454545454, "English,Chinese": 0.7386363636363636 }, "3_combine": { "Vietnamese,Indonesian,Spanish": 0.6306818181818182, "Vietnamese,Indonesian,Malay": 0.6193181818181818, "Vietnamese,Indonesian,Filipino": 0.6022727272727273, "Vietnamese,Indonesian,English": 0.6136363636363636, "Vietnamese,Indonesian,Chinese": 0.5681818181818182, "Vietnamese,Spanish,Malay": 0.6306818181818182, "Vietnamese,Spanish,Filipino": 0.6306818181818182, "Vietnamese,Spanish,English": 0.6363636363636364, "Vietnamese,Spanish,Chinese": 0.5852272727272727, "Vietnamese,Malay,Filipino": 0.5909090909090909, "Vietnamese,Malay,English": 0.6193181818181818, "Vietnamese,Malay,Chinese": 0.5454545454545454, "Vietnamese,Filipino,English": 0.6136363636363636, "Vietnamese,Filipino,Chinese": 0.5511363636363636, "Vietnamese,English,Chinese": 0.5795454545454546, "Indonesian,Spanish,Malay": 0.6704545454545454, "Indonesian,Spanish,Filipino": 0.6761363636363636, "Indonesian,Spanish,English": 0.6818181818181818, "Indonesian,Spanish,Chinese": 0.6306818181818182, "Indonesian,Malay,Filipino": 0.6534090909090909, "Indonesian,Malay,English": 0.6647727272727273, "Indonesian,Malay,Chinese": 0.5852272727272727, "Indonesian,Filipino,English": 0.6534090909090909, "Indonesian,Filipino,Chinese": 0.5965909090909091, "Indonesian,English,Chinese": 0.6193181818181818, "Spanish,Malay,Filipino": 0.6477272727272727, "Spanish,Malay,English": 0.6647727272727273, "Spanish,Malay,Chinese": 0.5738636363636364, "Spanish,Filipino,English": 0.6761363636363636, "Spanish,Filipino,Chinese": 0.6022727272727273, "Spanish,English,Chinese": 0.6306818181818182, "Malay,Filipino,English": 0.6477272727272727, "Malay,Filipino,Chinese": 0.5511363636363636, "Malay,English,Chinese": 0.5852272727272727, "Filipino,English,Chinese": 0.6079545454545454 }, "4_combine": { "Vietnamese,Indonesian,Spanish,Malay": 0.5738636363636364, "Vietnamese,Indonesian,Spanish,Filipino": 0.5795454545454546, "Vietnamese,Indonesian,Spanish,English": 0.5625, "Vietnamese,Indonesian,Spanish,Chinese": 0.5227272727272727, "Vietnamese,Indonesian,Malay,Filipino": 0.5511363636363636, "Vietnamese,Indonesian,Malay,English": 0.5625, "Vietnamese,Indonesian,Malay,Chinese": 0.5056818181818182, "Vietnamese,Indonesian,Filipino,English": 0.5454545454545454, "Vietnamese,Indonesian,Filipino,Chinese": 0.5113636363636364, "Vietnamese,Indonesian,English,Chinese": 0.5227272727272727, "Vietnamese,Spanish,Malay,Filipino": 0.5681818181818182, "Vietnamese,Spanish,Malay,English": 0.5795454545454546, "Vietnamese,Spanish,Malay,Chinese": 0.5113636363636364, "Vietnamese,Spanish,Filipino,English": 0.5795454545454546, "Vietnamese,Spanish,Filipino,Chinese": 0.5170454545454546, "Vietnamese,Spanish,English,Chinese": 0.5284090909090909, "Vietnamese,Malay,Filipino,English": 0.5397727272727273, "Vietnamese,Malay,Filipino,Chinese": 0.4943181818181818, "Vietnamese,Malay,English,Chinese": 0.5113636363636364, "Vietnamese,Filipino,English,Chinese": 0.5227272727272727, "Indonesian,Spanish,Malay,Filipino": 0.6079545454545454, "Indonesian,Spanish,Malay,English": 0.6079545454545454, "Indonesian,Spanish,Malay,Chinese": 0.5340909090909091, "Indonesian,Spanish,Filipino,English": 0.6079545454545454, "Indonesian,Spanish,Filipino,Chinese": 0.5625, "Indonesian,Spanish,English,Chinese": 0.5795454545454546, "Indonesian,Malay,Filipino,English": 0.5965909090909091, "Indonesian,Malay,Filipino,Chinese": 0.5227272727272727, "Indonesian,Malay,English,Chinese": 0.5397727272727273, "Indonesian,Filipino,English,Chinese": 0.5568181818181818, "Spanish,Malay,Filipino,English": 0.6079545454545454, "Spanish,Malay,Filipino,Chinese": 0.5284090909090909, "Spanish,Malay,English,Chinese": 0.5397727272727273, "Spanish,Filipino,English,Chinese": 0.5681818181818182, "Malay,Filipino,English,Chinese": 0.5340909090909091 }, "5_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino": 0.5397727272727273, "Vietnamese,Indonesian,Spanish,Malay,English": 0.5340909090909091, "Vietnamese,Indonesian,Spanish,Malay,Chinese": 0.4772727272727273, "Vietnamese,Indonesian,Spanish,Filipino,English": 0.5284090909090909, "Vietnamese,Indonesian,Spanish,Filipino,Chinese": 0.48863636363636365, "Vietnamese,Indonesian,Spanish,English,Chinese": 0.48863636363636365, "Vietnamese,Indonesian,Malay,Filipino,English": 0.5113636363636364, "Vietnamese,Indonesian,Malay,Filipino,Chinese": 0.4715909090909091, "Vietnamese,Indonesian,Malay,English,Chinese": 0.48295454545454547, "Vietnamese,Indonesian,Filipino,English,Chinese": 0.48863636363636365, "Vietnamese,Spanish,Malay,Filipino,English": 0.5340909090909091, "Vietnamese,Spanish,Malay,Filipino,Chinese": 0.48295454545454547, "Vietnamese,Spanish,Malay,English,Chinese": 0.48863636363636365, "Vietnamese,Spanish,Filipino,English,Chinese": 0.5, "Vietnamese,Malay,Filipino,English,Chinese": 0.4772727272727273, "Indonesian,Spanish,Malay,Filipino,English": 0.5681818181818182, "Indonesian,Spanish,Malay,Filipino,Chinese": 0.5056818181818182, "Indonesian,Spanish,Malay,English,Chinese": 0.5113636363636364, "Indonesian,Spanish,Filipino,English,Chinese": 0.5340909090909091, "Indonesian,Malay,Filipino,English,Chinese": 0.5056818181818182, "Spanish,Malay,Filipino,English,Chinese": 0.5170454545454546 }, "6_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English": 0.5056818181818182, "Vietnamese,Indonesian,Spanish,Malay,Filipino,Chinese": 0.4602272727272727, "Vietnamese,Indonesian,Spanish,Malay,English,Chinese": 0.4602272727272727, "Vietnamese,Indonesian,Spanish,Filipino,English,Chinese": 0.4715909090909091, "Vietnamese,Indonesian,Malay,Filipino,English,Chinese": 0.45454545454545453, "Vietnamese,Spanish,Malay,Filipino,English,Chinese": 0.4715909090909091, "Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.4943181818181818 }, "7_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.44886363636363635 } }, "AC3_2": 0.6908056813881713, "AC3_3": 0.6349235842859868, "AC3_4": 0.5974954368037646, "AC3_5": 0.5703372257502052, "AC3_6": 0.5491597738948918, "AC3_7": 0.5318885241025783 }, "prompt_3": { "overall_acc": 0.6542207792207793, "language_acc": { "Vietnamese": 0.5965909090909091, "Indonesian": 0.6420454545454546, "Spanish": 0.6761363636363636, "Malay": 0.6647727272727273, "Filipino": 0.6477272727272727, "English": 0.6988636363636364, "Chinese": 0.6534090909090909 }, "consistency_score_2": 0.7351190476190477, "consistency_score_3": 0.6209415584415584, "consistency_score_4": 0.5501623376623376, "consistency_score_5": 0.5, "consistency_score_6": 0.4618506493506494, "consistency_score_7": 0.4318181818181818, "detailed_consistency_score": { "2_combine": { "Vietnamese,Indonesian": 0.6818181818181818, "Vietnamese,Spanish": 0.6931818181818182, "Vietnamese,Malay": 0.6931818181818182, "Vietnamese,Filipino": 0.6761363636363636, "Vietnamese,English": 0.6988636363636364, "Vietnamese,Chinese": 0.6420454545454546, "Indonesian,Spanish": 0.7784090909090909, "Indonesian,Malay": 0.7840909090909091, "Indonesian,Filipino": 0.7556818181818182, "Indonesian,English": 0.7784090909090909, "Indonesian,Chinese": 0.7159090909090909, "Spanish,Malay": 0.7840909090909091, "Spanish,Filipino": 0.7784090909090909, "Spanish,English": 0.7784090909090909, "Spanish,Chinese": 0.7329545454545454, "Malay,Filipino": 0.7386363636363636, "Malay,English": 0.7556818181818182, "Malay,Chinese": 0.7386363636363636, "Filipino,English": 0.7727272727272727, "Filipino,Chinese": 0.7272727272727273, "English,Chinese": 0.7329545454545454 }, "3_combine": { "Vietnamese,Indonesian,Spanish": 0.5965909090909091, "Vietnamese,Indonesian,Malay": 0.5965909090909091, "Vietnamese,Indonesian,Filipino": 0.5738636363636364, "Vietnamese,Indonesian,English": 0.5965909090909091, "Vietnamese,Indonesian,Chinese": 0.5454545454545454, "Vietnamese,Spanish,Malay": 0.6136363636363636, "Vietnamese,Spanish,Filipino": 0.5965909090909091, "Vietnamese,Spanish,English": 0.6079545454545454, "Vietnamese,Spanish,Chinese": 0.5568181818181818, "Vietnamese,Malay,Filipino": 0.5795454545454546, "Vietnamese,Malay,English": 0.5965909090909091, "Vietnamese,Malay,Chinese": 0.5568181818181818, "Vietnamese,Filipino,English": 0.5852272727272727, "Vietnamese,Filipino,Chinese": 0.5454545454545454, "Vietnamese,English,Chinese": 0.5454545454545454, "Indonesian,Spanish,Malay": 0.6818181818181818, "Indonesian,Spanish,Filipino": 0.6704545454545454, "Indonesian,Spanish,English": 0.6875, "Indonesian,Spanish,Chinese": 0.6306818181818182, "Indonesian,Malay,Filipino": 0.6590909090909091, "Indonesian,Malay,English": 0.6761363636363636, "Indonesian,Malay,Chinese": 0.6363636363636364, "Indonesian,Filipino,English": 0.6647727272727273, "Indonesian,Filipino,Chinese": 0.6193181818181818, "Indonesian,English,Chinese": 0.625, "Spanish,Malay,Filipino": 0.6704545454545454, "Spanish,Malay,English": 0.6818181818181818, "Spanish,Malay,Chinese": 0.6363636363636364, "Spanish,Filipino,English": 0.6761363636363636, "Spanish,Filipino,Chinese": 0.6363636363636364, "Spanish,English,Chinese": 0.6420454545454546, "Malay,Filipino,English": 0.6590909090909091, "Malay,Filipino,Chinese": 0.625, "Malay,English,Chinese": 0.625, "Filipino,English,Chinese": 0.6363636363636364 }, "4_combine": { "Vietnamese,Indonesian,Spanish,Malay": 0.5568181818181818, "Vietnamese,Indonesian,Spanish,Filipino": 0.5340909090909091, "Vietnamese,Indonesian,Spanish,English": 0.5511363636363636, "Vietnamese,Indonesian,Spanish,Chinese": 0.5056818181818182, "Vietnamese,Indonesian,Malay,Filipino": 0.5340909090909091, "Vietnamese,Indonesian,Malay,English": 0.5511363636363636, "Vietnamese,Indonesian,Malay,Chinese": 0.5, "Vietnamese,Indonesian,Filipino,English": 0.5170454545454546, "Vietnamese,Indonesian,Filipino,Chinese": 0.4943181818181818, "Vietnamese,Indonesian,English,Chinese": 0.4943181818181818, "Vietnamese,Spanish,Malay,Filipino": 0.5454545454545454, "Vietnamese,Spanish,Malay,English": 0.5625, "Vietnamese,Spanish,Malay,Chinese": 0.5170454545454546, "Vietnamese,Spanish,Filipino,English": 0.5454545454545454, "Vietnamese,Spanish,Filipino,Chinese": 0.5, "Vietnamese,Spanish,English,Chinese": 0.5, "Vietnamese,Malay,Filipino,English": 0.5454545454545454, "Vietnamese,Malay,Filipino,Chinese": 0.5056818181818182, "Vietnamese,Malay,English,Chinese": 0.5056818181818182, "Vietnamese,Filipino,English,Chinese": 0.48863636363636365, "Indonesian,Spanish,Malay,Filipino": 0.6136363636363636, "Indonesian,Spanish,Malay,English": 0.6306818181818182, "Indonesian,Spanish,Malay,Chinese": 0.5738636363636364, "Indonesian,Spanish,Filipino,English": 0.6079545454545454, "Indonesian,Spanish,Filipino,Chinese": 0.5738636363636364, "Indonesian,Spanish,English,Chinese": 0.5852272727272727, "Indonesian,Malay,Filipino,English": 0.6079545454545454, "Indonesian,Malay,Filipino,Chinese": 0.5625, "Indonesian,Malay,English,Chinese": 0.5738636363636364, "Indonesian,Filipino,English,Chinese": 0.5568181818181818, "Spanish,Malay,Filipino,English": 0.6079545454545454, "Spanish,Malay,Filipino,Chinese": 0.5795454545454546, "Spanish,Malay,English,Chinese": 0.5795454545454546, "Spanish,Filipino,English,Chinese": 0.5738636363636364, "Malay,Filipino,English,Chinese": 0.5738636363636364 }, "5_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino": 0.5113636363636364, "Vietnamese,Indonesian,Spanish,Malay,English": 0.5284090909090909, "Vietnamese,Indonesian,Spanish,Malay,Chinese": 0.4772727272727273, "Vietnamese,Indonesian,Spanish,Filipino,English": 0.4943181818181818, "Vietnamese,Indonesian,Spanish,Filipino,Chinese": 0.4659090909090909, "Vietnamese,Indonesian,Spanish,English,Chinese": 0.4715909090909091, "Vietnamese,Indonesian,Malay,Filipino,English": 0.5056818181818182, "Vietnamese,Indonesian,Malay,Filipino,Chinese": 0.4659090909090909, "Vietnamese,Indonesian,Malay,English,Chinese": 0.4715909090909091, "Vietnamese,Indonesian,Filipino,English,Chinese": 0.44886363636363635, "Vietnamese,Spanish,Malay,Filipino,English": 0.5170454545454546, "Vietnamese,Spanish,Malay,Filipino,Chinese": 0.4772727272727273, "Vietnamese,Spanish,Malay,English,Chinese": 0.48295454545454547, "Vietnamese,Spanish,Filipino,English,Chinese": 0.4602272727272727, "Vietnamese,Malay,Filipino,English,Chinese": 0.4772727272727273, "Indonesian,Spanish,Malay,Filipino,English": 0.5738636363636364, "Indonesian,Spanish,Malay,Filipino,Chinese": 0.5340909090909091, "Indonesian,Spanish,Malay,English,Chinese": 0.5454545454545454, "Indonesian,Spanish,Filipino,English,Chinese": 0.5284090909090909, "Indonesian,Malay,Filipino,English,Chinese": 0.5284090909090909, "Spanish,Malay,Filipino,English,Chinese": 0.5340909090909091 }, "6_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English": 0.48863636363636365, "Vietnamese,Indonesian,Spanish,Malay,Filipino,Chinese": 0.44886363636363635, "Vietnamese,Indonesian,Spanish,Malay,English,Chinese": 0.4602272727272727, "Vietnamese,Indonesian,Spanish,Filipino,English,Chinese": 0.4318181818181818, "Vietnamese,Indonesian,Malay,Filipino,English,Chinese": 0.4431818181818182, "Vietnamese,Spanish,Malay,Filipino,English,Chinese": 0.45454545454545453, "Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.5056818181818182 }, "7_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.4318181818181818 } }, "AC3_2": 0.6923146473281081, "AC3_3": 0.6371469077844022, "AC3_4": 0.5976962449373581, "AC3_5": 0.5668073135936493, "AC3_6": 0.5414569066811196, "AC3_7": 0.5202473161635386 }, "prompt_4": { "overall_acc": 0.6590909090909091, "language_acc": { "Vietnamese": 0.6193181818181818, "Indonesian": 0.6306818181818182, "Spanish": 0.6647727272727273, "Malay": 0.6590909090909091, "Filipino": 0.6420454545454546, "English": 0.7329545454545454, "Chinese": 0.6647727272727273 }, "consistency_score_2": 0.7372835497835497, "consistency_score_3": 0.6230519480519479, "consistency_score_4": 0.552435064935065, "consistency_score_5": 0.5021645021645023, "consistency_score_6": 0.4634740259740259, "consistency_score_7": 0.4318181818181818, "detailed_consistency_score": { "2_combine": { "Vietnamese,Indonesian": 0.6988636363636364, "Vietnamese,Spanish": 0.75, "Vietnamese,Malay": 0.7045454545454546, "Vietnamese,Filipino": 0.7159090909090909, "Vietnamese,English": 0.75, "Vietnamese,Chinese": 0.6477272727272727, "Indonesian,Spanish": 0.7727272727272727, "Indonesian,Malay": 0.7613636363636364, "Indonesian,Filipino": 0.7443181818181818, "Indonesian,English": 0.7727272727272727, "Indonesian,Chinese": 0.7159090909090909, "Spanish,Malay": 0.75, "Spanish,Filipino": 0.7954545454545454, "Spanish,English": 0.8409090909090909, "Spanish,Chinese": 0.7272727272727273, "Malay,Filipino": 0.7329545454545454, "Malay,English": 0.7556818181818182, "Malay,Chinese": 0.6818181818181818, "Filipino,English": 0.7784090909090909, "Filipino,Chinese": 0.6931818181818182, "English,Chinese": 0.6931818181818182 }, "3_combine": { "Vietnamese,Indonesian,Spanish": 0.625, "Vietnamese,Indonesian,Malay": 0.5965909090909091, "Vietnamese,Indonesian,Filipino": 0.6022727272727273, "Vietnamese,Indonesian,English": 0.6193181818181818, "Vietnamese,Indonesian,Chinese": 0.5568181818181818, "Vietnamese,Spanish,Malay": 0.6193181818181818, "Vietnamese,Spanish,Filipino": 0.6534090909090909, "Vietnamese,Spanish,English": 0.6818181818181818, "Vietnamese,Spanish,Chinese": 0.5738636363636364, "Vietnamese,Malay,Filipino": 0.5965909090909091, "Vietnamese,Malay,English": 0.6193181818181818, "Vietnamese,Malay,Chinese": 0.5511363636363636, "Vietnamese,Filipino,English": 0.6363636363636364, "Vietnamese,Filipino,Chinese": 0.5511363636363636, "Vietnamese,English,Chinese": 0.5568181818181818, "Indonesian,Spanish,Malay": 0.6647727272727273, "Indonesian,Spanish,Filipino": 0.6704545454545454, "Indonesian,Spanish,English": 0.7045454545454546, "Indonesian,Spanish,Chinese": 0.6306818181818182, "Indonesian,Malay,Filipino": 0.6420454545454546, "Indonesian,Malay,English": 0.6534090909090909, "Indonesian,Malay,Chinese": 0.6022727272727273, "Indonesian,Filipino,English": 0.6647727272727273, "Indonesian,Filipino,Chinese": 0.6022727272727273, "Indonesian,English,Chinese": 0.6136363636363636, "Spanish,Malay,Filipino": 0.6477272727272727, "Spanish,Malay,English": 0.6875, "Spanish,Malay,Chinese": 0.6022727272727273, "Spanish,Filipino,English": 0.7159090909090909, "Spanish,Filipino,Chinese": 0.6193181818181818, "Spanish,English,Chinese": 0.6420454545454546, "Malay,Filipino,English": 0.6477272727272727, "Malay,Filipino,Chinese": 0.5681818181818182, "Malay,English,Chinese": 0.5852272727272727, "Filipino,English,Chinese": 0.6022727272727273 }, "4_combine": { "Vietnamese,Indonesian,Spanish,Malay": 0.5511363636363636, "Vietnamese,Indonesian,Spanish,Filipino": 0.5738636363636364, "Vietnamese,Indonesian,Spanish,English": 0.5909090909090909, "Vietnamese,Indonesian,Spanish,Chinese": 0.5227272727272727, "Vietnamese,Indonesian,Malay,Filipino": 0.5397727272727273, "Vietnamese,Indonesian,Malay,English": 0.5625, "Vietnamese,Indonesian,Malay,Chinese": 0.5, "Vietnamese,Indonesian,Filipino,English": 0.5568181818181818, "Vietnamese,Indonesian,Filipino,Chinese": 0.5056818181818182, "Vietnamese,Indonesian,English,Chinese": 0.5056818181818182, "Vietnamese,Spanish,Malay,Filipino": 0.5625, "Vietnamese,Spanish,Malay,English": 0.5965909090909091, "Vietnamese,Spanish,Malay,Chinese": 0.5056818181818182, "Vietnamese,Spanish,Filipino,English": 0.6136363636363636, "Vietnamese,Spanish,Filipino,Chinese": 0.5227272727272727, "Vietnamese,Spanish,English,Chinese": 0.5284090909090909, "Vietnamese,Malay,Filipino,English": 0.5625, "Vietnamese,Malay,Filipino,Chinese": 0.48863636363636365, "Vietnamese,Malay,English,Chinese": 0.5056818181818182, "Vietnamese,Filipino,English,Chinese": 0.5056818181818182, "Indonesian,Spanish,Malay,Filipino": 0.5909090909090909, "Indonesian,Spanish,Malay,English": 0.6193181818181818, "Indonesian,Spanish,Malay,Chinese": 0.5568181818181818, "Indonesian,Spanish,Filipino,English": 0.625, "Indonesian,Spanish,Filipino,Chinese": 0.5625, "Indonesian,Spanish,English,Chinese": 0.5738636363636364, "Indonesian,Malay,Filipino,English": 0.5852272727272727, "Indonesian,Malay,Filipino,Chinese": 0.5284090909090909, "Indonesian,Malay,English,Chinese": 0.5454545454545454, "Indonesian,Filipino,English,Chinese": 0.5511363636363636, "Spanish,Malay,Filipino,English": 0.6136363636363636, "Spanish,Malay,Filipino,Chinese": 0.5284090909090909, "Spanish,Malay,English,Chinese": 0.5625, "Spanish,Filipino,English,Chinese": 0.5681818181818182, "Malay,Filipino,English,Chinese": 0.5227272727272727 }, "5_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino": 0.5170454545454546, "Vietnamese,Indonesian,Spanish,Malay,English": 0.5397727272727273, "Vietnamese,Indonesian,Spanish,Malay,Chinese": 0.4715909090909091, "Vietnamese,Indonesian,Spanish,Filipino,English": 0.5454545454545454, "Vietnamese,Indonesian,Spanish,Filipino,Chinese": 0.48863636363636365, "Vietnamese,Indonesian,Spanish,English,Chinese": 0.48863636363636365, "Vietnamese,Indonesian,Malay,Filipino,English": 0.5170454545454546, "Vietnamese,Indonesian,Malay,Filipino,Chinese": 0.4602272727272727, "Vietnamese,Indonesian,Malay,English,Chinese": 0.4772727272727273, "Vietnamese,Indonesian,Filipino,English,Chinese": 0.4715909090909091, "Vietnamese,Spanish,Malay,Filipino,English": 0.5511363636363636, "Vietnamese,Spanish,Malay,Filipino,Chinese": 0.4659090909090909, "Vietnamese,Spanish,Malay,English,Chinese": 0.48863636363636365, "Vietnamese,Spanish,Filipino,English,Chinese": 0.48863636363636365, "Vietnamese,Malay,Filipino,English,Chinese": 0.4659090909090909, "Indonesian,Spanish,Malay,Filipino,English": 0.5625, "Indonesian,Spanish,Malay,Filipino,Chinese": 0.5, "Indonesian,Spanish,Malay,English,Chinese": 0.5227272727272727, "Indonesian,Spanish,Filipino,English,Chinese": 0.5227272727272727, "Indonesian,Malay,Filipino,English,Chinese": 0.4943181818181818, "Spanish,Malay,Filipino,English,Chinese": 0.5056818181818182 }, "6_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English": 0.5056818181818182, "Vietnamese,Indonesian,Spanish,Malay,Filipino,Chinese": 0.4431818181818182, "Vietnamese,Indonesian,Spanish,Malay,English,Chinese": 0.4602272727272727, "Vietnamese,Indonesian,Spanish,Filipino,English,Chinese": 0.4602272727272727, "Vietnamese,Indonesian,Malay,Filipino,English,Chinese": 0.4431818181818182, "Vietnamese,Spanish,Malay,Filipino,English,Chinese": 0.45454545454545453, "Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.4772727272727273 }, "7_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.4318181818181818 } }, "AC3_2": 0.6959979566534031, "AC3_3": 0.6405649300819485, "AC3_4": 0.6010683005363083, "AC3_5": 0.5700245699754831, "AC3_6": 0.5442384801986754, "AC3_7": 0.5217803029824732 }, "prompt_5": { "overall_acc": 0.661525974025974, "language_acc": { "Vietnamese": 0.6136363636363636, "Indonesian": 0.6647727272727273, "Spanish": 0.6931818181818182, "Malay": 0.6193181818181818, "Filipino": 0.6193181818181818, "English": 0.7272727272727273, "Chinese": 0.6931818181818182 }, "consistency_score_2": 0.7394480519480519, "consistency_score_3": 0.6262987012987011, "consistency_score_4": 0.558116883116883, "consistency_score_5": 0.5113636363636362, "consistency_score_6": 0.476461038961039, "consistency_score_7": 0.44886363636363635, "detailed_consistency_score": { "2_combine": { "Vietnamese,Indonesian": 0.7215909090909091, "Vietnamese,Spanish": 0.7727272727272727, "Vietnamese,Malay": 0.6931818181818182, "Vietnamese,Filipino": 0.7045454545454546, "Vietnamese,English": 0.7159090909090909, "Vietnamese,Chinese": 0.6875, "Indonesian,Spanish": 0.7897727272727273, "Indonesian,Malay": 0.7670454545454546, "Indonesian,Filipino": 0.7386363636363636, "Indonesian,English": 0.7329545454545454, "Indonesian,Chinese": 0.7272727272727273, "Spanish,Malay": 0.7727272727272727, "Spanish,Filipino": 0.7954545454545454, "Spanish,English": 0.8181818181818182, "Spanish,Chinese": 0.7556818181818182, "Malay,Filipino": 0.7329545454545454, "Malay,English": 0.7329545454545454, "Malay,Chinese": 0.6988636363636364, "Filipino,English": 0.7727272727272727, "Filipino,Chinese": 0.6988636363636364, "English,Chinese": 0.6988636363636364 }, "3_combine": { "Vietnamese,Indonesian,Spanish": 0.6534090909090909, "Vietnamese,Indonesian,Malay": 0.6079545454545454, "Vietnamese,Indonesian,Filipino": 0.6022727272727273, "Vietnamese,Indonesian,English": 0.6022727272727273, "Vietnamese,Indonesian,Chinese": 0.5852272727272727, "Vietnamese,Spanish,Malay": 0.6363636363636364, "Vietnamese,Spanish,Filipino": 0.6477272727272727, "Vietnamese,Spanish,English": 0.6647727272727273, "Vietnamese,Spanish,Chinese": 0.6193181818181818, "Vietnamese,Malay,Filipino": 0.5795454545454546, "Vietnamese,Malay,English": 0.5909090909090909, "Vietnamese,Malay,Chinese": 0.5625, "Vietnamese,Filipino,English": 0.6193181818181818, "Vietnamese,Filipino,Chinese": 0.5681818181818182, "Vietnamese,English,Chinese": 0.5681818181818182, "Indonesian,Spanish,Malay": 0.6761363636363636, "Indonesian,Spanish,Filipino": 0.6761363636363636, "Indonesian,Spanish,English": 0.6875, "Indonesian,Spanish,Chinese": 0.6534090909090909, "Indonesian,Malay,Filipino": 0.6420454545454546, "Indonesian,Malay,English": 0.625, "Indonesian,Malay,Chinese": 0.6136363636363636, "Indonesian,Filipino,English": 0.6363636363636364, "Indonesian,Filipino,Chinese": 0.6079545454545454, "Indonesian,English,Chinese": 0.5965909090909091, "Spanish,Malay,Filipino": 0.6647727272727273, "Spanish,Malay,English": 0.6761363636363636, "Spanish,Malay,Chinese": 0.6306818181818182, "Spanish,Filipino,English": 0.7102272727272727, "Spanish,Filipino,Chinese": 0.6363636363636364, "Spanish,English,Chinese": 0.6534090909090909, "Malay,Filipino,English": 0.6420454545454546, "Malay,Filipino,Chinese": 0.5795454545454546, "Malay,English,Chinese": 0.5909090909090909, "Filipino,English,Chinese": 0.6136363636363636 }, "4_combine": { "Vietnamese,Indonesian,Spanish,Malay": 0.5738636363636364, "Vietnamese,Indonesian,Spanish,Filipino": 0.5738636363636364, "Vietnamese,Indonesian,Spanish,English": 0.5681818181818182, "Vietnamese,Indonesian,Spanish,Chinese": 0.5625, "Vietnamese,Indonesian,Malay,Filipino": 0.5284090909090909, "Vietnamese,Indonesian,Malay,English": 0.5340909090909091, "Vietnamese,Indonesian,Malay,Chinese": 0.5340909090909091, "Vietnamese,Indonesian,Filipino,English": 0.5454545454545454, "Vietnamese,Indonesian,Filipino,Chinese": 0.5227272727272727, "Vietnamese,Indonesian,English,Chinese": 0.5113636363636364, "Vietnamese,Spanish,Malay,Filipino": 0.5511363636363636, "Vietnamese,Spanish,Malay,English": 0.5681818181818182, "Vietnamese,Spanish,Malay,Chinese": 0.5454545454545454, "Vietnamese,Spanish,Filipino,English": 0.5965909090909091, "Vietnamese,Spanish,Filipino,Chinese": 0.5340909090909091, "Vietnamese,Spanish,English,Chinese": 0.5511363636363636, "Vietnamese,Malay,Filipino,English": 0.5397727272727273, "Vietnamese,Malay,Filipino,Chinese": 0.5, "Vietnamese,Malay,English,Chinese": 0.5, "Vietnamese,Filipino,English,Chinese": 0.5284090909090909, "Indonesian,Spanish,Malay,Filipino": 0.6079545454545454, "Indonesian,Spanish,Malay,English": 0.6079545454545454, "Indonesian,Spanish,Malay,Chinese": 0.5909090909090909, "Indonesian,Spanish,Filipino,English": 0.6136363636363636, "Indonesian,Spanish,Filipino,Chinese": 0.5681818181818182, "Indonesian,Spanish,English,Chinese": 0.5795454545454546, "Indonesian,Malay,Filipino,English": 0.5852272727272727, "Indonesian,Malay,Filipino,Chinese": 0.5397727272727273, "Indonesian,Malay,English,Chinese": 0.5397727272727273, "Indonesian,Filipino,English,Chinese": 0.5511363636363636, "Spanish,Malay,Filipino,English": 0.625, "Spanish,Malay,Filipino,Chinese": 0.5625, "Spanish,Malay,English,Chinese": 0.5625, "Spanish,Filipino,English,Chinese": 0.5909090909090909, "Malay,Filipino,English,Chinese": 0.5397727272727273 }, "5_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino": 0.5113636363636364, "Vietnamese,Indonesian,Spanish,Malay,English": 0.5170454545454546, "Vietnamese,Indonesian,Spanish,Malay,Chinese": 0.5227272727272727, "Vietnamese,Indonesian,Spanish,Filipino,English": 0.5284090909090909, "Vietnamese,Indonesian,Spanish,Filipino,Chinese": 0.5, "Vietnamese,Indonesian,Spanish,English,Chinese": 0.5, "Vietnamese,Indonesian,Malay,Filipino,English": 0.5, "Vietnamese,Indonesian,Malay,Filipino,Chinese": 0.4772727272727273, "Vietnamese,Indonesian,Malay,English,Chinese": 0.4772727272727273, "Vietnamese,Indonesian,Filipino,English,Chinese": 0.48863636363636365, "Vietnamese,Spanish,Malay,Filipino,English": 0.5284090909090909, "Vietnamese,Spanish,Malay,Filipino,Chinese": 0.48863636363636365, "Vietnamese,Spanish,Malay,English,Chinese": 0.4943181818181818, "Vietnamese,Spanish,Filipino,English,Chinese": 0.5113636363636364, "Vietnamese,Malay,Filipino,English,Chinese": 0.4772727272727273, "Indonesian,Spanish,Malay,Filipino,English": 0.5738636363636364, "Indonesian,Spanish,Malay,Filipino,Chinese": 0.5284090909090909, "Indonesian,Spanish,Malay,English,Chinese": 0.5340909090909091, "Indonesian,Spanish,Filipino,English,Chinese": 0.5340909090909091, "Indonesian,Malay,Filipino,English,Chinese": 0.5113636363636364, "Spanish,Malay,Filipino,English,Chinese": 0.5340909090909091 }, "6_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English": 0.48863636363636365, "Vietnamese,Indonesian,Spanish,Malay,Filipino,Chinese": 0.4659090909090909, "Vietnamese,Indonesian,Spanish,Malay,English,Chinese": 0.4715909090909091, "Vietnamese,Indonesian,Spanish,Filipino,English,Chinese": 0.4772727272727273, "Vietnamese,Indonesian,Malay,Filipino,English,Chinese": 0.45454545454545453, "Vietnamese,Spanish,Malay,Filipino,English,Chinese": 0.4715909090909091, "Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.5056818181818182 }, "7_combine": { "Vietnamese,Indonesian,Spanish,Malay,Filipino,English,Chinese": 0.44886363636363635 } }, "AC3_2": 0.6983200026589176, "AC3_3": 0.643430532609109, "AC3_4": 0.6054375877851255, "AC3_5": 0.5768323371637796, "AC3_6": 0.5539454304124446, "AC3_7": 0.5348302099465176 } }, "sg_eval": { "prompt_1": { "accuracy": 0.7766990291262136 }, "prompt_2": { "accuracy": 0.8155339805825242 }, "prompt_3": { "accuracy": 0.7961165048543689 }, "prompt_4": { "accuracy": 0.8058252427184466 }, "prompt_5": { "accuracy": 0.7669902912621359 } }, "cn_eval": { "prompt_1": { "accuracy": 0.638095238095238 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.6095238095238096 }, "prompt_4": { "accuracy": 0.5619047619047619 }, "prompt_5": { "accuracy": 0.6571428571428571 } }, "us_eval": { "prompt_1": { "accuracy": 0.8785046728971962 }, "prompt_2": { "accuracy": 0.8785046728971962 }, "prompt_3": { "accuracy": 0.9065420560747663 }, "prompt_4": { "accuracy": 0.8598130841121495 }, "prompt_5": { "accuracy": 0.897196261682243 } }, "ph_eval": { "prompt_1": { "accuracy": 0.77, "category_acc": { "brand": 0.6, "demographics": 0.6, "biology": 0.8, "history": 0.8, "literature": 0.8, "politics": 0.8, "culture": 1.0, "film": 0.5, "law": 0.8, "geography": 0.9 } }, "prompt_2": { "accuracy": 0.79, "category_acc": { "brand": 0.5, "demographics": 0.4, "biology": 0.9, "history": 0.8666666666666667, "literature": 0.8, "politics": 0.9, "culture": 1.0, "film": 0.6, "law": 0.8, "geography": 0.9 } }, "prompt_3": { "accuracy": 0.78, "category_acc": { "brand": 0.7, "demographics": 0.2, "biology": 0.9, "history": 0.8, "literature": 0.8, "politics": 0.8, "culture": 1.0, "film": 0.6, "law": 0.8, "geography": 0.9 } }, "prompt_4": { "accuracy": 0.77, "category_acc": { "brand": 0.5, "demographics": 0.4, "biology": 0.8, "history": 0.7333333333333333, "literature": 0.8, "politics": 0.8, "culture": 1.0, "film": 0.7, "law": 0.9, "geography": 0.9 } }, "prompt_5": { "accuracy": 0.79, "category_acc": { "brand": 0.7, "demographics": 0.6, "biology": 0.7, "history": 0.8666666666666667, "literature": 0.8, "politics": 0.9, "culture": 1.0, "film": 0.5, "law": 0.7, "geography": 1.0 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.25051857825738894 }, "prompt_2": { "bleu_score": 0.2488096986298307 }, "prompt_3": { "bleu_score": 0.26375119990462953 }, "prompt_4": { "bleu_score": 0.2620911518816617 }, "prompt_5": { "bleu_score": 0.22352841486416786 } }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "gemma_2b": { "model_size": ">2B", "model_link": "https://huggingface.co/google/gemma-2b", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.5142857142857142, "language_acc": { "Spanish": 0.5109243697478991, "English": 0.5462184873949579, "Chinese": 0.5100840336134453, "Vietnamese": 0.4899159663865546 }, "consistency_score_2": 0.707843137254902, "consistency_score_3": 0.5827731092436975, "consistency_score_4": 0.5067226890756302, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7420168067226891, "Spanish,Chinese": 0.6882352941176471, "Spanish,Vietnamese": 0.7084033613445379, "English,Chinese": 0.7067226890756303, "English,Vietnamese": 0.7168067226890756, "Chinese,Vietnamese": 0.6848739495798319 }, "3_combine": { "Spanish,English,Chinese": 0.5907563025210084, "Spanish,English,Vietnamese": 0.6, "Spanish,Chinese,Vietnamese": 0.5596638655462185, "English,Chinese,Vietnamese": 0.5806722689075631 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5067226890756302 } }, "AC3_2": 0.5957368782460054, "AC3_3": 0.5463916397160207, "AC3_4": 0.5104761904261932 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.38095238095238093, "language_acc": { "Filipino": 0.3466666666666667, "Vietnamese": 0.35333333333333333, "Chinese": 0.36666666666666664, "Spanish": 0.42, "Malay": 0.34, "Indonesian": 0.4, "English": 0.44 }, "consistency_score_2": 0.5517460317460318, "consistency_score_3": 0.3777142857142858, "consistency_score_4": 0.28400000000000003, "consistency_score_5": 0.22412698412698417, "consistency_score_6": 0.18285714285714283, "consistency_score_7": 0.15333333333333332, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5, "Filipino,Chinese": 0.5133333333333333, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.5533333333333333, "Filipino,Indonesian": 0.5466666666666666, "Filipino,English": 0.5266666666666666, "Vietnamese,Chinese": 0.5066666666666667, "Vietnamese,Spanish": 0.5466666666666666, "Vietnamese,Malay": 0.5533333333333333, "Vietnamese,Indonesian": 0.5533333333333333, "Vietnamese,English": 0.5066666666666667, "Chinese,Spanish": 0.54, "Chinese,Malay": 0.54, "Chinese,Indonesian": 0.5666666666666667, "Chinese,English": 0.56, "Spanish,Malay": 0.6, "Spanish,Indonesian": 0.5866666666666667, "Spanish,English": 0.6133333333333333, "Malay,Indonesian": 0.6, "Malay,English": 0.5933333333333334, "Indonesian,English": 0.6 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.31333333333333335, "Filipino,Vietnamese,Spanish": 0.34, "Filipino,Vietnamese,Malay": 0.35333333333333333, "Filipino,Vietnamese,Indonesian": 0.35333333333333333, "Filipino,Vietnamese,English": 0.32666666666666666, "Filipino,Chinese,Spanish": 0.3333333333333333, "Filipino,Chinese,Malay": 0.38, "Filipino,Chinese,Indonesian": 0.35333333333333333, "Filipino,Chinese,English": 0.37333333333333335, "Filipino,Spanish,Malay": 0.35333333333333333, "Filipino,Spanish,Indonesian": 0.35333333333333333, "Filipino,Spanish,English": 0.36666666666666664, "Filipino,Malay,Indonesian": 0.4, "Filipino,Malay,English": 0.38, "Filipino,Indonesian,English": 0.38666666666666666, "Vietnamese,Chinese,Spanish": 0.36666666666666664, "Vietnamese,Chinese,Malay": 0.3466666666666667, "Vietnamese,Chinese,Indonesian": 0.36666666666666664, "Vietnamese,Chinese,English": 0.34, "Vietnamese,Spanish,Malay": 0.3933333333333333, "Vietnamese,Spanish,Indonesian": 0.4, "Vietnamese,Spanish,English": 0.38666666666666666, "Vietnamese,Malay,Indonesian": 0.4, "Vietnamese,Malay,English": 0.36666666666666664, "Vietnamese,Indonesian,English": 0.37333333333333335, "Chinese,Spanish,Malay": 0.38, "Chinese,Spanish,Indonesian": 0.3933333333333333, "Chinese,Spanish,English": 0.4, "Chinese,Malay,Indonesian": 0.4, "Chinese,Malay,English": 0.4, "Chinese,Indonesian,English": 0.4066666666666667, "Spanish,Malay,Indonesian": 0.42, "Spanish,Malay,English": 0.44666666666666666, "Spanish,Indonesian,English": 0.43333333333333335, "Malay,Indonesian,English": 0.43333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.26, "Filipino,Vietnamese,Chinese,Indonesian": 0.25333333333333335, "Filipino,Vietnamese,Chinese,English": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Spanish,English": 0.2733333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Malay,English": 0.26, "Filipino,Vietnamese,Indonesian,English": 0.26, "Filipino,Chinese,Spanish,Malay": 0.26666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.26, "Filipino,Chinese,Spanish,English": 0.2733333333333333, "Filipino,Chinese,Malay,Indonesian": 0.2866666666666667, "Filipino,Chinese,Malay,English": 0.29333333333333333, "Filipino,Chinese,Indonesian,English": 0.28, "Filipino,Spanish,Malay,Indonesian": 0.28, "Filipino,Spanish,Malay,English": 0.28, "Filipino,Spanish,Indonesian,English": 0.29333333333333333, "Filipino,Malay,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.2866666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.3, "Vietnamese,Chinese,Spanish,English": 0.28, "Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, "Vietnamese,Chinese,Malay,English": 0.26666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.2733333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.32, "Vietnamese,Spanish,Malay,English": 0.31333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, "Vietnamese,Malay,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay,Indonesian": 0.3, "Chinese,Spanish,Malay,English": 0.32, "Chinese,Spanish,Indonesian,English": 0.31333333333333335, "Chinese,Malay,Indonesian,English": 0.30666666666666664, "Spanish,Malay,Indonesian,English": 0.32666666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.2, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Chinese,Malay,English": 0.2, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.22, "Filipino,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay,English": 0.24, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, "Vietnamese,Chinese,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Spanish,Malay,Indonesian,English": 0.26666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 } }, "AC3_2": 0.4507115303092246, "AC3_3": 0.379326422962088, "AC3_4": 0.32540819244605007, "AC3_5": 0.28221655918985505, "AC3_6": 0.24710424706041945, "AC3_7": 0.21865715979270525 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3133116883116883, "language_acc": { "Indonesian": 0.2840909090909091, "English": 0.35795454545454547, "Filipino": 0.2897727272727273, "Spanish": 0.30113636363636365, "Chinese": 0.35795454545454547, "Malay": 0.30113636363636365, "Vietnamese": 0.30113636363636365 }, "consistency_score_2": 0.5097402597402597, "consistency_score_3": 0.3136363636363637, "consistency_score_4": 0.21250000000000002, "consistency_score_5": 0.1523268398268398, "consistency_score_6": 0.11282467532467533, "consistency_score_7": 0.08522727272727272, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5056818181818182, "Indonesian,Filipino": 0.5909090909090909, "Indonesian,Spanish": 0.5113636363636364, "Indonesian,Chinese": 0.44886363636363635, "Indonesian,Malay": 0.5568181818181818, "Indonesian,Vietnamese": 0.5113636363636364, "English,Filipino": 0.5170454545454546, "English,Spanish": 0.6363636363636364, "English,Chinese": 0.48863636363636365, "English,Malay": 0.4772727272727273, "English,Vietnamese": 0.4715909090909091, "Filipino,Spanish": 0.5284090909090909, "Filipino,Chinese": 0.4659090909090909, "Filipino,Malay": 0.5284090909090909, "Filipino,Vietnamese": 0.4715909090909091, "Spanish,Chinese": 0.5284090909090909, "Spanish,Malay": 0.5340909090909091, "Spanish,Vietnamese": 0.5397727272727273, "Chinese,Malay": 0.4772727272727273, "Chinese,Vietnamese": 0.45454545454545453, "Malay,Vietnamese": 0.4602272727272727 }, "3_combine": { "Indonesian,English,Filipino": 0.36363636363636365, "Indonesian,English,Spanish": 0.375, "Indonesian,English,Chinese": 0.2727272727272727, "Indonesian,English,Malay": 0.32954545454545453, "Indonesian,English,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Spanish": 0.35795454545454547, "Indonesian,Filipino,Chinese": 0.2897727272727273, "Indonesian,Filipino,Malay": 0.375, "Indonesian,Filipino,Vietnamese": 0.3352272727272727, "Indonesian,Spanish,Chinese": 0.29545454545454547, "Indonesian,Spanish,Malay": 0.3352272727272727, "Indonesian,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,Chinese,Malay": 0.2897727272727273, "Indonesian,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,Malay,Vietnamese": 0.3181818181818182, "English,Filipino,Spanish": 0.3806818181818182, "English,Filipino,Chinese": 0.2840909090909091, "English,Filipino,Malay": 0.3125, "English,Filipino,Vietnamese": 0.2840909090909091, "English,Spanish,Chinese": 0.35795454545454547, "English,Spanish,Malay": 0.36363636363636365, "English,Spanish,Vietnamese": 0.35795454545454547, "English,Chinese,Malay": 0.2840909090909091, "English,Chinese,Vietnamese": 0.2556818181818182, "English,Malay,Vietnamese": 0.2784090909090909, "Filipino,Spanish,Chinese": 0.30113636363636365, "Filipino,Spanish,Malay": 0.32954545454545453, "Filipino,Spanish,Vietnamese": 0.3181818181818182, "Filipino,Chinese,Malay": 0.2784090909090909, "Filipino,Chinese,Vietnamese": 0.2556818181818182, "Filipino,Malay,Vietnamese": 0.29545454545454547, "Spanish,Chinese,Malay": 0.32386363636363635, "Spanish,Chinese,Vietnamese": 0.3125, "Spanish,Malay,Vietnamese": 0.32386363636363635, "Chinese,Malay,Vietnamese": 0.24431818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2784090909090909, "Indonesian,English,Filipino,Chinese": 0.19886363636363635, "Indonesian,English,Filipino,Malay": 0.25, "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,English,Spanish,Chinese": 0.21022727272727273, "Indonesian,English,Spanish,Malay": 0.26704545454545453, "Indonesian,English,Spanish,Vietnamese": 0.25, "Indonesian,English,Chinese,Malay": 0.21022727272727273, "Indonesian,English,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,English,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.19318181818181818, "Indonesian,Filipino,Spanish,Malay": 0.23295454545454544, "Indonesian,Filipino,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Chinese,Malay": 0.19318181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Chinese,Malay": 0.2159090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Chinese,Malay,Vietnamese": 0.1875, "English,Filipino,Spanish,Chinese": 0.21022727272727273, "English,Filipino,Spanish,Malay": 0.25, "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, "English,Filipino,Chinese,Malay": 0.19318181818181818, "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, "English,Filipino,Malay,Vietnamese": 0.1875, "English,Spanish,Chinese,Malay": 0.23863636363636365, "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, "English,Spanish,Malay,Vietnamese": 0.22727272727272727, "English,Chinese,Malay,Vietnamese": 0.16477272727272727, "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.1875, "Indonesian,English,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.125, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 } }, "AC3_2": 0.3880862726545052, "AC3_3": 0.31347394185480676, "AC3_4": 0.2532417412300102, "AC3_5": 0.20499068039697646, "AC3_6": 0.16590599872420903, "AC3_7": 0.13400296238074416 } }, "sg_eval": { "prompt_1": { "accuracy": 0.39805825242718446 } }, "cn_eval": { "prompt_1": { "accuracy": 0.2761904761904762 } }, "us_eval": { "prompt_1": { "accuracy": 0.3644859813084112 } }, "ph_eval": { "prompt_1": { "accuracy": 0.28, "category_acc": { "brand": 0.1, "demographics": 0.0, "biology": 0.1, "history": 0.13333333333333333, "literature": 0.2, "politics": 0.5, "culture": 0.4, "film": 0.3, "law": 0.5, "geography": 0.5 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.2954373835094491 } }, "indommlu": { "prompt_1": { "accuracy": 0.3148407770879231, "category_acc": { "History": 0.28714859437751006, "Geography": 0.2938775510204082, "Lampungic": 0.30612244897959184, "Social science": 0.4273789649415693, "Balinese": 0.27388535031847133, "Makassarese": 0.3279569892473118, "Banjarese": 0.3402777777777778, "Chemistry": 0.24087591240875914, "Biology": 0.29704142011834317, "Science": 0.3560371517027864, "Christian religion": 0.32338308457711445, "Art": 0.3427620632279534, "Islam religion": 0.3328591749644381, "Hindu religion": 0.26666666666666666, "Madurese": 0.24745762711864408, "Sport": 0.2972972972972973, "Indonesian language": 0.34526774595267745, "Physics": 0.29494949494949496, "Minangkabau culture": 0.2663316582914573, "Dayak language": 0.1926605504587156, "Sociology": 0.2721774193548387, "Economy": 0.26844262295081966, "Sundanese": 0.31201382886776147, "Javanese": 0.26713709677419356, "Civic education": 0.35050071530758226 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.3387755038281587 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.26946302062889876 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.21693912788544986 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.3359460665195331 } }, "mmlu": { "prompt_1": { "accuracy": 0.3757292882147025 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.3841258491240615, "category_acc": { "high_school_european_history": 0.43902439024390244, "business_ethics": 0.36363636363636365, "clinical_knowledge": 0.4318181818181818, "medical_genetics": 0.43434343434343436, "high_school_us_history": 0.4630541871921182, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.4703389830508475, "virology": 0.34545454545454546, "high_school_microeconomics": 0.3670886075949367, "econometrics": 0.20353982300884957, "college_computer_science": 0.23232323232323232, "high_school_biology": 0.44336569579288027, "abstract_algebra": 0.2828282828282828, "professional_accounting": 0.3594306049822064, "philosophy": 0.43870967741935485, "professional_medicine": 0.2693726937269373, "nutrition": 0.4098360655737705, "global_facts": 0.2828282828282828, "machine_learning": 0.36936936936936937, "security_studies": 0.45081967213114754, "public_relations": 0.41284403669724773, "professional_psychology": 0.37152209492635024, "prehistory": 0.4458204334365325, "anatomy": 0.43283582089552236, "human_sexuality": 0.47692307692307695, "college_medicine": 0.37790697674418605, "high_school_government_and_politics": 0.4427083333333333, "college_chemistry": 0.2828282828282828, "logical_fallacies": 0.3888888888888889, "high_school_geography": 0.40609137055837563, "elementary_mathematics": 0.27586206896551724, "human_aging": 0.4369369369369369, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.4577205882352941, "formal_logic": 0.272, "high_school_statistics": 0.22790697674418606, "international_law": 0.5083333333333333, "high_school_mathematics": 0.26022304832713755, "high_school_computer_science": 0.3333333333333333, "conceptual_physics": 0.3888888888888889, "miscellaneous": 0.5281329923273658, "high_school_chemistry": 0.32673267326732675, "marketing": 0.5407725321888412, "professional_law": 0.33202870189171557, "management": 0.4411764705882353, "college_physics": 0.25742574257425743, "jurisprudence": 0.42990654205607476, "world_religions": 0.5294117647058824, "sociology": 0.5, "us_foreign_policy": 0.5050505050505051, "high_school_macroeconomics": 0.34704370179948585, "computer_security": 0.5050505050505051, "moral_scenarios": 0.24720357941834453, "moral_disputes": 0.4463768115942029, "electrical_engineering": 0.4722222222222222, "astronomy": 0.4105960264900662, "college_biology": 0.3916083916083916 } } }, "c_eval": { "prompt_1": { "accuracy": 0.33580980683506684 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.32503113325031135, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.25, "computer_architecture": 0.38461538461538464, "college_programming": 0.38095238095238093, "college_physics": 0.2916666666666667, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.3333333333333333, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.375, "high_school_chemistry": 0.4583333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.08333333333333333, "middle_school_biology": 0.4230769230769231, "middle_school_physics": 0.5, "middle_school_chemistry": 0.44, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.38333333333333336, "business_administration": 0.3157894736842105, "marxism": 0.375, "mao_zedong_thought": 0.3448275862068966, "education_science": 0.29411764705882354, "teacher_qualification": 0.3469387755102041, "high_school_politics": 0.5, "high_school_geography": 0.25, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.375, "logic": 0.3333333333333333, "law": 0.2413793103448276, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.2894736842105263, "professional_tour_guide": 0.4117647058823529, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.36, "middle_school_history": 0.25925925925925924, "civil_servant": 0.2692307692307692, "sports_science": 0.3333333333333333, "plant_protection": 0.37037037037037035, "basic_medicine": 0.375, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.39215686274509803, "accountant": 0.35185185185185186, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.24074074074074073, "physician": 0.2777777777777778 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.3225806451612903 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.31246762217233637, "category_acc": { "agronomy": 0.21893491124260356, "anatomy": 0.2702702702702703, "ancient_chinese": 0.2865853658536585, "arts": 0.35, "astronomy": 0.296969696969697, "business_ethics": 0.3253588516746411, "chinese_civil_service_exam": 0.31875, "chinese_driving_rule": 0.4198473282442748, "chinese_food_culture": 0.3088235294117647, "chinese_foreign_policy": 0.2897196261682243, "chinese_history": 0.28173374613003094, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.2849162011173184, "clinical_knowledge": 0.27848101265822783, "college_actuarial_science": 0.22641509433962265, "college_education": 0.308411214953271, "college_engineering_hydrology": 0.3584905660377358, "college_law": 0.26851851851851855, "college_mathematics": 0.21904761904761905, "college_medical_statistics": 0.39622641509433965, "college_medicine": 0.2857142857142857, "computer_science": 0.3431372549019608, "computer_security": 0.32748538011695905, "conceptual_physics": 0.3129251700680272, "construction_project_management": 0.3597122302158273, "economics": 0.3584905660377358, "education": 0.2883435582822086, "electrical_engineering": 0.3488372093023256, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.32323232323232326, "elementary_information_and_technology": 0.4579831932773109, "elementary_mathematics": 0.2565217391304348, "ethnology": 0.2740740740740741, "food_science": 0.34265734265734266, "genetics": 0.3125, "global_facts": 0.31543624161073824, "high_school_biology": 0.2958579881656805, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.2542372881355932, "high_school_mathematics": 0.20121951219512196, "high_school_physics": 0.3181818181818182, "high_school_politics": 0.36363636363636365, "human_sexuality": 0.3333333333333333, "international_law": 0.2918918918918919, "journalism": 0.32558139534883723, "jurisprudence": 0.32116788321167883, "legal_and_moral_basis": 0.48598130841121495, "logical": 0.2764227642276423, "machine_learning": 0.26229508196721313, "management": 0.3476190476190476, "marketing": 0.36666666666666664, "marxist_theory": 0.2698412698412698, "modern_chinese": 0.28448275862068967, "nutrition": 0.2689655172413793, "philosophy": 0.3238095238095238, "professional_accounting": 0.30857142857142855, "professional_law": 0.2890995260663507, "professional_medicine": 0.26063829787234044, "professional_psychology": 0.36637931034482757, "public_relations": 0.39080459770114945, "security_study": 0.362962962962963, "sociology": 0.3141592920353982, "sports_science": 0.34545454545454546, "traditional_chinese_medicine": 0.2756756756756757, "virology": 0.3136094674556213, "world_history": 0.30434782608695654, "world_religions": 0.30625 } } }, "zbench": { "prompt_1": { "accuracy": 0.30303030303030304 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.21818181818181817 } }, "ocnli": { "prompt_1": { "accuracy": 0.3298305084745763 } }, "c3": { "prompt_1": { "accuracy": 0.4682124158563949 } }, "dream": { "prompt_1": { "accuracy": 0.5766780989710926 } }, "samsum": { "prompt_1": { "rouge1": 0.3982816908544844, "rouge2": 0.14738674876970356, "rougeL": 0.31698064657325636, "avg_rouge": 0.2875496953991481 } }, "dialogsum": { "prompt_1": { "rouge1": 0.3848274473994049, "rouge2": 0.1376326801953284, "rougeL": 0.31241519162134024, "avg_rouge": 0.2782917730720245 } }, "sst2": { "prompt_1": { "accuracy": 0.6559633027522935 } }, "cola": { "prompt_1": { "accuracy": 0.6682646212847555 } }, "qqp": { "prompt_1": { "accuracy": 0.5095 } }, "mnli": { "prompt_1": { "accuracy": 0.2 } }, "qnli": { "prompt_1": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.4 } }, "rte": { "prompt_1": { "accuracy": 0.5 } }, "mrpc": { "prompt_1": { "accuracy": 0.8 } } } }, "gemma_7b": { "model_size": "7B", "model_link": "https://huggingface.co/google/gemma-7b", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9239495798319327, "language_acc": { "Spanish": 0.9235294117647059, "English": 0.9453781512605042, "Chinese": 0.915126050420168, "Vietnamese": 0.9117647058823529 }, "consistency_score_2": 0.9155462184873951, "consistency_score_3": 0.8764705882352941, "consistency_score_4": 0.8478991596638655, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.938655462184874, "Spanish,Chinese": 0.9067226890756303, "Spanish,Vietnamese": 0.9084033613445378, "English,Chinese": 0.9243697478991597, "English,Vietnamese": 0.9176470588235294, "Chinese,Vietnamese": 0.8974789915966387 }, "3_combine": { "Spanish,English,Chinese": 0.8873949579831932, "Spanish,English,Vietnamese": 0.8840336134453781, "Spanish,Chinese,Vietnamese": 0.8605042016806723, "English,Chinese,Vietnamese": 0.8739495798319328 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8478991596638655 } }, "AC3_2": 0.919728704588549, "AC3_3": 0.8995840482718138, "AC3_4": 0.8842922703322786 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.6447619047619048, "language_acc": { "Filipino": 0.6266666666666667, "Vietnamese": 0.6133333333333333, "Chinese": 0.6266666666666667, "Spanish": 0.6733333333333333, "Malay": 0.6266666666666667, "Indonesian": 0.6533333333333333, "English": 0.6933333333333334 }, "consistency_score_2": 0.7069841269841268, "consistency_score_3": 0.575047619047619, "consistency_score_4": 0.4939047619047618, "consistency_score_5": 0.4371428571428572, "consistency_score_6": 0.39428571428571424, "consistency_score_7": 0.36, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.66, "Filipino,Chinese": 0.6933333333333334, "Filipino,Spanish": 0.68, "Filipino,Malay": 0.72, "Filipino,Indonesian": 0.7066666666666667, "Filipino,English": 0.7333333333333333, "Vietnamese,Chinese": 0.62, "Vietnamese,Spanish": 0.6866666666666666, "Vietnamese,Malay": 0.7133333333333334, "Vietnamese,Indonesian": 0.6866666666666666, "Vietnamese,English": 0.7066666666666667, "Chinese,Spanish": 0.7, "Chinese,Malay": 0.6666666666666666, "Chinese,Indonesian": 0.6666666666666666, "Chinese,English": 0.7, "Spanish,Malay": 0.7466666666666667, "Spanish,Indonesian": 0.7733333333333333, "Spanish,English": 0.78, "Malay,Indonesian": 0.7133333333333334, "Malay,English": 0.7466666666666667, "Indonesian,English": 0.7466666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.5, "Filipino,Vietnamese,Spanish": 0.5266666666666666, "Filipino,Vietnamese,Malay": 0.5666666666666667, "Filipino,Vietnamese,Indonesian": 0.54, "Filipino,Vietnamese,English": 0.56, "Filipino,Chinese,Spanish": 0.56, "Filipino,Chinese,Malay": 0.56, "Filipino,Chinese,Indonesian": 0.5533333333333333, "Filipino,Chinese,English": 0.5733333333333334, "Filipino,Spanish,Malay": 0.5933333333333334, "Filipino,Spanish,Indonesian": 0.6, "Filipino,Spanish,English": 0.6, "Filipino,Malay,Indonesian": 0.5866666666666667, "Filipino,Malay,English": 0.6133333333333333, "Filipino,Indonesian,English": 0.6133333333333333, "Vietnamese,Chinese,Spanish": 0.5266666666666666, "Vietnamese,Chinese,Malay": 0.52, "Vietnamese,Chinese,Indonesian": 0.5, "Vietnamese,Chinese,English": 0.5266666666666666, "Vietnamese,Spanish,Malay": 0.5866666666666667, "Vietnamese,Spanish,Indonesian": 0.5866666666666667, "Vietnamese,Spanish,English": 0.5933333333333334, "Vietnamese,Malay,Indonesian": 0.5666666666666667, "Vietnamese,Malay,English": 0.6, "Vietnamese,Indonesian,English": 0.5866666666666667, "Chinese,Spanish,Malay": 0.5666666666666667, "Chinese,Spanish,Indonesian": 0.5866666666666667, "Chinese,Spanish,English": 0.6, "Chinese,Malay,Indonesian": 0.54, "Chinese,Malay,English": 0.5666666666666667, "Chinese,Indonesian,English": 0.5733333333333334, "Spanish,Malay,Indonesian": 0.6333333333333333, "Spanish,Malay,English": 0.64, "Spanish,Indonesian,English": 0.66, "Malay,Indonesian,English": 0.62 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.43333333333333335, "Filipino,Vietnamese,Chinese,Malay": 0.44666666666666666, "Filipino,Vietnamese,Chinese,Indonesian": 0.43333333333333335, "Filipino,Vietnamese,Chinese,English": 0.44, "Filipino,Vietnamese,Spanish,Malay": 0.4866666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.4666666666666667, "Filipino,Vietnamese,Spanish,English": 0.4866666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.49333333333333335, "Filipino,Vietnamese,Malay,English": 0.5133333333333333, "Filipino,Vietnamese,Indonesian,English": 0.49333333333333335, "Filipino,Chinese,Spanish,Malay": 0.5, "Filipino,Chinese,Spanish,Indonesian": 0.5, "Filipino,Chinese,Spanish,English": 0.5, "Filipino,Chinese,Malay,Indonesian": 0.4866666666666667, "Filipino,Chinese,Malay,English": 0.5, "Filipino,Chinese,Indonesian,English": 0.5, "Filipino,Spanish,Malay,Indonesian": 0.5333333333333333, "Filipino,Spanish,Malay,English": 0.5333333333333333, "Filipino,Spanish,Indonesian,English": 0.5466666666666666, "Filipino,Malay,Indonesian,English": 0.54, "Vietnamese,Chinese,Spanish,Malay": 0.4666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.44666666666666666, "Vietnamese,Chinese,Spanish,English": 0.4866666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.43333333333333335, "Vietnamese,Chinese,Malay,English": 0.4666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.47333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.5066666666666667, "Vietnamese,Spanish,Malay,English": 0.5266666666666666, "Vietnamese,Spanish,Indonesian,English": 0.52, "Vietnamese,Malay,Indonesian,English": 0.5066666666666667, "Chinese,Spanish,Malay,Indonesian": 0.49333333333333335, "Chinese,Spanish,Malay,English": 0.52, "Chinese,Spanish,Indonesian,English": 0.5266666666666666, "Chinese,Malay,Indonesian,English": 0.5, "Spanish,Malay,Indonesian,English": 0.58 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.4066666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.4, "Filipino,Vietnamese,Chinese,Malay,English": 0.41333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4066666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.44, "Filipino,Vietnamese,Spanish,Malay,English": 0.4533333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.44, "Filipino,Vietnamese,Malay,Indonesian,English": 0.46, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.4533333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.46, "Filipino,Chinese,Spanish,Indonesian,English": 0.46, "Filipino,Chinese,Malay,Indonesian,English": 0.46, "Filipino,Spanish,Malay,Indonesian,English": 0.5, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4, "Vietnamese,Chinese,Spanish,Malay,English": 0.44, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.43333333333333335, "Vietnamese,Chinese,Malay,Indonesian,English": 0.42, "Vietnamese,Spanish,Malay,Indonesian,English": 0.4666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.47333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36666666666666664, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.37333333333333335, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.38666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.42, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36 } }, "AC3_2": 0.6744409402529532, "AC3_3": 0.6079126140901353, "AC3_4": 0.5593401200738682, "AC3_5": 0.5210311870745777, "AC3_6": 0.48933350787486446, "AC3_7": 0.4620284359729735 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.5771103896103896, "language_acc": { "Indonesian": 0.5681818181818182, "English": 0.625, "Filipino": 0.5227272727272727, "Spanish": 0.5738636363636364, "Chinese": 0.6022727272727273, "Malay": 0.5852272727272727, "Vietnamese": 0.5625 }, "consistency_score_2": 0.6696428571428572, "consistency_score_3": 0.5321428571428571, "consistency_score_4": 0.450974025974026, "consistency_score_5": 0.39529220779220775, "consistency_score_6": 0.3563311688311689, "consistency_score_7": 0.32954545454545453, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.7159090909090909, "Indonesian,Filipino": 0.6363636363636364, "Indonesian,Spanish": 0.6761363636363636, "Indonesian,Chinese": 0.6534090909090909, "Indonesian,Malay": 0.7443181818181818, "Indonesian,Vietnamese": 0.6534090909090909, "English,Filipino": 0.6363636363636364, "English,Spanish": 0.7272727272727273, "English,Chinese": 0.6818181818181818, "English,Malay": 0.75, "English,Vietnamese": 0.7045454545454546, "Filipino,Spanish": 0.625, "Filipino,Chinese": 0.5909090909090909, "Filipino,Malay": 0.625, "Filipino,Vietnamese": 0.6306818181818182, "Spanish,Chinese": 0.6931818181818182, "Spanish,Malay": 0.7045454545454546, "Spanish,Vietnamese": 0.6590909090909091, "Chinese,Malay": 0.6477272727272727, "Chinese,Vietnamese": 0.625, "Malay,Vietnamese": 0.6818181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.5227272727272727, "Indonesian,English,Spanish": 0.5795454545454546, "Indonesian,English,Chinese": 0.5568181818181818, "Indonesian,English,Malay": 0.625, "Indonesian,English,Vietnamese": 0.5568181818181818, "Indonesian,Filipino,Spanish": 0.5, "Indonesian,Filipino,Chinese": 0.4715909090909091, "Indonesian,Filipino,Malay": 0.5227272727272727, "Indonesian,Filipino,Vietnamese": 0.48863636363636365, "Indonesian,Spanish,Chinese": 0.5454545454545454, "Indonesian,Spanish,Malay": 0.5909090909090909, "Indonesian,Spanish,Vietnamese": 0.5170454545454546, "Indonesian,Chinese,Malay": 0.5625, "Indonesian,Chinese,Vietnamese": 0.4943181818181818, "Indonesian,Malay,Vietnamese": 0.5625, "English,Filipino,Spanish": 0.5227272727272727, "English,Filipino,Chinese": 0.48863636363636365, "English,Filipino,Malay": 0.5227272727272727, "English,Filipino,Vietnamese": 0.5056818181818182, "English,Spanish,Chinese": 0.5852272727272727, "English,Spanish,Malay": 0.6079545454545454, "English,Spanish,Vietnamese": 0.5568181818181818, "English,Chinese,Malay": 0.5625, "English,Chinese,Vietnamese": 0.5227272727272727, "English,Malay,Vietnamese": 0.5795454545454546, "Filipino,Spanish,Chinese": 0.4943181818181818, "Filipino,Spanish,Malay": 0.5056818181818182, "Filipino,Spanish,Vietnamese": 0.48863636363636365, "Filipino,Chinese,Malay": 0.4715909090909091, "Filipino,Chinese,Vietnamese": 0.4602272727272727, "Filipino,Malay,Vietnamese": 0.4943181818181818, "Spanish,Chinese,Malay": 0.5681818181818182, "Spanish,Chinese,Vietnamese": 0.5284090909090909, "Spanish,Malay,Vietnamese": 0.5511363636363636, "Chinese,Malay,Vietnamese": 0.5113636363636364 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.4431818181818182, "Indonesian,English,Filipino,Chinese": 0.42045454545454547, "Indonesian,English,Filipino,Malay": 0.4602272727272727, "Indonesian,English,Filipino,Vietnamese": 0.42613636363636365, "Indonesian,English,Spanish,Chinese": 0.5, "Indonesian,English,Spanish,Malay": 0.5284090909090909, "Indonesian,English,Spanish,Vietnamese": 0.4602272727272727, "Indonesian,English,Chinese,Malay": 0.5113636363636364, "Indonesian,English,Chinese,Vietnamese": 0.44886363636363635, "Indonesian,English,Malay,Vietnamese": 0.5056818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.42045454545454547, "Indonesian,Filipino,Spanish,Malay": 0.4431818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Chinese,Malay": 0.42045454545454547, "Indonesian,Filipino,Chinese,Vietnamese": 0.3977272727272727, "Indonesian,Filipino,Malay,Vietnamese": 0.4375, "Indonesian,Spanish,Chinese,Malay": 0.4943181818181818, "Indonesian,Spanish,Chinese,Vietnamese": 0.4318181818181818, "Indonesian,Spanish,Malay,Vietnamese": 0.4715909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.44886363636363635, "English,Filipino,Spanish,Chinese": 0.44886363636363635, "English,Filipino,Spanish,Malay": 0.4602272727272727, "English,Filipino,Spanish,Vietnamese": 0.4318181818181818, "English,Filipino,Chinese,Malay": 0.4318181818181818, "English,Filipino,Chinese,Vietnamese": 0.4034090909090909, "English,Filipino,Malay,Vietnamese": 0.42613636363636365, "English,Spanish,Chinese,Malay": 0.5113636363636364, "English,Spanish,Chinese,Vietnamese": 0.4772727272727273, "English,Spanish,Malay,Vietnamese": 0.48863636363636365, "English,Chinese,Malay,Vietnamese": 0.4602272727272727, "Filipino,Spanish,Chinese,Malay": 0.42613636363636365, "Filipino,Spanish,Chinese,Vietnamese": 0.42613636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.4318181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.4034090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.4659090909090909 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.39204545454545453, "Indonesian,English,Filipino,Spanish,Malay": 0.4090909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.375, "Indonesian,English,Filipino,Chinese,Malay": 0.39204545454545453, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.35795454545454547, "Indonesian,English,Filipino,Malay,Vietnamese": 0.38636363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.4659090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4090909090909091, "Indonesian,English,Spanish,Malay,Vietnamese": 0.4318181818181818, "Indonesian,English,Chinese,Malay,Vietnamese": 0.42045454545454547, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3806818181818182, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.3693181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.38636363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.36363636363636365, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4034090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.4034090909090909, "English,Filipino,Spanish,Chinese,Vietnamese": 0.39204545454545453, "English,Filipino,Spanish,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Chinese,Malay,Vietnamese": 0.36363636363636365, "English,Spanish,Chinese,Malay,Vietnamese": 0.4318181818181818, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3806818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3693181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.3465909090909091, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.3522727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3352272727272727, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.39204545454545453, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3409090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.35795454545454547 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.32954545454545453 } }, "AC3_2": 0.6199428012895613, "AC3_3": 0.5537151637559821, "AC3_4": 0.506304330390564, "AC3_5": 0.4692032716383252, "AC3_6": 0.44061123654229467, "AC3_7": 0.41952877019870227 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6990291262135923 } }, "cn_eval": { "prompt_1": { "accuracy": 0.5428571428571428 } }, "us_eval": { "prompt_1": { "accuracy": 0.7663551401869159 } }, "ph_eval": { "prompt_1": { "accuracy": 0.61, "category_acc": { "brand": 0.6, "demographics": 0.2, "biology": 0.5, "history": 0.6666666666666666, "literature": 0.5, "politics": 0.8, "culture": 0.9, "film": 0.4, "law": 0.7, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.3675732291585539 } }, "indommlu": { "prompt_1": { "accuracy": 0.5378863742572936, "category_acc": { "History": 0.5020080321285141, "Geography": 0.49183673469387756, "Lampungic": 0.2925170068027211, "Social science": 0.7779632721202003, "Balinese": 0.3524416135881104, "Makassarese": 0.3763440860215054, "Banjarese": 0.4305555555555556, "Chemistry": 0.36934306569343067, "Biology": 0.5514792899408284, "Science": 0.6893704850361198, "Christian religion": 0.6616915422885572, "Art": 0.6256239600665557, "Islam religion": 0.6258890469416786, "Hindu religion": 0.5266666666666666, "Madurese": 0.31186440677966104, "Sport": 0.5608108108108109, "Indonesian language": 0.6014943960149439, "Physics": 0.45454545454545453, "Minangkabau culture": 0.4321608040201005, "Dayak language": 0.3302752293577982, "Sociology": 0.5221774193548387, "Economy": 0.5020491803278688, "Sundanese": 0.45030250648228176, "Javanese": 0.42036290322580644, "Civic education": 0.6409155937052933 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.4174588580163382 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.35063626935110476 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.2709583870415398 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.4255264854611502 } }, "mmlu": { "prompt_1": { "accuracy": 0.6487747957992999 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.636968180193064, "category_acc": { "high_school_european_history": 0.7865853658536586, "business_ethics": 0.6464646464646465, "clinical_knowledge": 0.7045454545454546, "medical_genetics": 0.7272727272727273, "high_school_us_history": 0.8275862068965517, "high_school_physics": 0.3933333333333333, "high_school_world_history": 0.8347457627118644, "virology": 0.5515151515151515, "high_school_microeconomics": 0.6497890295358649, "econometrics": 0.4778761061946903, "college_computer_science": 0.5555555555555556, "high_school_biology": 0.7928802588996764, "abstract_algebra": 0.30303030303030304, "professional_accounting": 0.505338078291815, "philosophy": 0.7225806451612903, "professional_medicine": 0.7011070110701108, "nutrition": 0.7442622950819672, "global_facts": 0.40404040404040403, "machine_learning": 0.44144144144144143, "security_studies": 0.7049180327868853, "public_relations": 0.6880733944954128, "professional_psychology": 0.6693944353518821, "prehistory": 0.7461300309597523, "anatomy": 0.6044776119402985, "human_sexuality": 0.7461538461538462, "college_medicine": 0.6627906976744186, "high_school_government_and_politics": 0.875, "college_chemistry": 0.5050505050505051, "logical_fallacies": 0.7160493827160493, "high_school_geography": 0.8071065989847716, "elementary_mathematics": 0.4350132625994695, "human_aging": 0.6801801801801802, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.8584558823529411, "formal_logic": 0.472, "high_school_statistics": 0.5581395348837209, "international_law": 0.7583333333333333, "high_school_mathematics": 0.42379182156133827, "high_school_computer_science": 0.6666666666666666, "conceptual_physics": 0.6239316239316239, "miscellaneous": 0.850383631713555, "high_school_chemistry": 0.5346534653465347, "marketing": 0.8755364806866953, "professional_law": 0.48532289628180036, "management": 0.8529411764705882, "college_physics": 0.43564356435643564, "jurisprudence": 0.7850467289719626, "world_religions": 0.8705882352941177, "sociology": 0.855, "us_foreign_policy": 0.8484848484848485, "high_school_macroeconomics": 0.6632390745501285, "computer_security": 0.7373737373737373, "moral_scenarios": 0.2494407158836689, "moral_disputes": 0.6869565217391305, "electrical_engineering": 0.6319444444444444, "astronomy": 0.7284768211920529, "college_biology": 0.7482517482517482 } } }, "c_eval": { "prompt_1": { "accuracy": 0.4985141158989599 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.5024906600249066, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5, "computer_architecture": 0.5769230769230769, "college_programming": 0.5, "college_physics": 0.25, "college_chemistry": 0.4482758620689655, "advanced_mathematics": 0.4583333333333333, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.35714285714285715, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.375, "high_school_chemistry": 0.5, "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.8461538461538461, "middle_school_physics": 0.625, "middle_school_chemistry": 0.68, "veterinary_medicine": 0.7142857142857143, "college_economics": 0.5166666666666667, "business_administration": 0.34210526315789475, "marxism": 0.4583333333333333, "mao_zedong_thought": 0.5172413793103449, "education_science": 0.6176470588235294, "teacher_qualification": 0.673469387755102, "high_school_politics": 0.5, "high_school_geography": 0.625, "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.5882352941176471, "modern_chinese_history": 0.6071428571428571, "ideological_and_moral_cultivation": 0.625, "logic": 0.5555555555555556, "law": 0.41379310344827586, "chinese_language_and_literature": 0.5357142857142857, "art_studies": 0.6052631578947368, "professional_tour_guide": 0.5588235294117647, "legal_professional": 0.39285714285714285, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.6, "middle_school_history": 0.5185185185185185, "civil_servant": 0.5, "sports_science": 0.4583333333333333, "plant_protection": 0.7037037037037037, "basic_medicine": 0.4583333333333333, "clinical_medicine": 0.5185185185185185, "urban_and_rural_planner": 0.5294117647058824, "accountant": 0.35185185185185186, "fire_engineer": 0.4166666666666667, "environmental_impact_assessment_engineer": 0.4722222222222222, "tax_accountant": 0.37037037037037035, "physician": 0.5740740740740741 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.5268817204301075 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.5088931099982732, "category_acc": { "agronomy": 0.4911242603550296, "anatomy": 0.3716216216216216, "ancient_chinese": 0.2804878048780488, "arts": 0.63125, "astronomy": 0.36363636363636365, "business_ethics": 0.5311004784688995, "chinese_civil_service_exam": 0.425, "chinese_driving_rule": 0.648854961832061, "chinese_food_culture": 0.3897058823529412, "chinese_foreign_policy": 0.5887850467289719, "chinese_history": 0.5510835913312694, "chinese_literature": 0.29901960784313725, "chinese_teacher_qualification": 0.6368715083798883, "clinical_knowledge": 0.4430379746835443, "college_actuarial_science": 0.2641509433962264, "college_education": 0.5794392523364486, "college_engineering_hydrology": 0.5754716981132075, "college_law": 0.46296296296296297, "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.5754716981132075, "college_medicine": 0.48717948717948717, "computer_science": 0.553921568627451, "computer_security": 0.6900584795321637, "conceptual_physics": 0.5306122448979592, "construction_project_management": 0.45323741007194246, "economics": 0.5911949685534591, "education": 0.558282208588957, "electrical_engineering": 0.5058139534883721, "elementary_chinese": 0.3373015873015873, "elementary_commonsense": 0.5505050505050505, "elementary_information_and_technology": 0.7941176470588235, "elementary_mathematics": 0.34347826086956523, "ethnology": 0.4666666666666667, "food_science": 0.5804195804195804, "genetics": 0.5227272727272727, "global_facts": 0.5436241610738255, "high_school_biology": 0.47337278106508873, "high_school_chemistry": 0.38636363636363635, "high_school_geography": 0.4915254237288136, "high_school_mathematics": 0.3231707317073171, "high_school_physics": 0.36363636363636365, "high_school_politics": 0.5104895104895105, "human_sexuality": 0.5079365079365079, "international_law": 0.42162162162162165, "journalism": 0.6104651162790697, "jurisprudence": 0.5109489051094891, "legal_and_moral_basis": 0.8411214953271028, "logical": 0.5365853658536586, "machine_learning": 0.5327868852459017, "management": 0.6, "marketing": 0.5833333333333334, "marxist_theory": 0.6349206349206349, "modern_chinese": 0.45689655172413796, "nutrition": 0.5517241379310345, "philosophy": 0.5428571428571428, "professional_accounting": 0.5542857142857143, "professional_law": 0.38388625592417064, "professional_medicine": 0.39095744680851063, "professional_psychology": 0.521551724137931, "public_relations": 0.5804597701149425, "security_study": 0.6888888888888889, "sociology": 0.5442477876106194, "sports_science": 0.48484848484848486, "traditional_chinese_medicine": 0.3891891891891892, "virology": 0.6094674556213018, "world_history": 0.5652173913043478, "world_religions": 0.54375 } } }, "zbench": { "prompt_1": { "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.6068181818181818 } }, "ocnli": { "prompt_1": { "accuracy": 0.43627118644067797 } }, "c3": { "prompt_1": { "accuracy": 0.8537771129394166 } }, "dream": { "prompt_1": { "accuracy": 0.910828025477707 } }, "samsum": { "prompt_1": { "rouge1": 0.454330568867136, "rouge2": 0.21266277688547977, "rougeL": 0.37251449141327697, "avg_rouge": 0.34650261238863095 } }, "dialogsum": { "prompt_1": { "rouge1": 0.4224311816685506, "rouge2": 0.17731176655538933, "rougeL": 0.3507821111092207, "avg_rouge": 0.31684168644438687 } }, "sst2": { "prompt_1": { "accuracy": 0.9495412844036697 } }, "cola": { "prompt_1": { "accuracy": 0.6375838926174496 } }, "qqp": { "prompt_1": { "accuracy": 0.652 } }, "mnli": { "prompt_1": { "accuracy": 0.6 } }, "qnli": { "prompt_1": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.9 } }, "rte": { "prompt_1": { "accuracy": 0.8 } }, "mrpc": { "prompt_1": { "accuracy": 0.8 } } } }, "gemma_2b_it": { "model_size": ">2B", "model_link": "https://huggingface.co/google/gemma-2b-it", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.6340336134453781, "language_acc": { "Spanish": 0.6235294117647059, "English": 0.6672268907563025, "Chinese": 0.6319327731092437, "Vietnamese": 0.6134453781512605 }, "consistency_score_2": 0.7525210084033613, "consistency_score_3": 0.6483193277310924, "consistency_score_4": 0.5882352941176471, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7915966386554621, "Spanish,Chinese": 0.7369747899159664, "Spanish,Vietnamese": 0.7563025210084033, "English,Chinese": 0.746218487394958, "English,Vietnamese": 0.7546218487394958, "Chinese,Vietnamese": 0.7294117647058823 }, "3_combine": { "Spanish,English,Chinese": 0.6571428571428571, "Spanish,English,Vietnamese": 0.6680672268907563, "Spanish,Chinese,Vietnamese": 0.6310924369747899, "English,Chinese,Vietnamese": 0.6369747899159663 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5882352941176471 } }, "AC3_2": 0.688214667635621, "AC3_3": 0.6410968974245923, "AC3_4": 0.6102764240294991 }, "prompt_2": { "overall_acc": 0.6756302521008403, "language_acc": { "Spanish": 0.6554621848739496, "English": 0.7067226890756303, "Chinese": 0.673109243697479, "Vietnamese": 0.6672268907563025 }, "consistency_score_2": 0.7473389355742297, "consistency_score_3": 0.6409663865546218, "consistency_score_4": 0.5756302521008403, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7739495798319328, "Spanish,Chinese": 0.7184873949579832, "Spanish,Vietnamese": 0.7546218487394958, "English,Chinese": 0.7512605042016807, "English,Vietnamese": 0.7630252100840336, "Chinese,Vietnamese": 0.7226890756302521 }, "3_combine": { "Spanish,English,Chinese": 0.6420168067226891, "Spanish,English,Vietnamese": 0.661344537815126, "Spanish,Chinese,Vietnamese": 0.6210084033613446, "English,Chinese,Vietnamese": 0.6394957983193277 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5756302521008403 } }, "AC3_2": 0.7096777608182537, "AC3_3": 0.6578419974479328, "AC3_4": 0.6216342816011932 }, "prompt_3": { "overall_acc": 0.6771008403361344, "language_acc": { "Spanish": 0.6621848739495798, "English": 0.7100840336134454, "Chinese": 0.673109243697479, "Vietnamese": 0.6630252100840336 }, "consistency_score_2": 0.7470588235294118, "consistency_score_3": 0.6411764705882352, "consistency_score_4": 0.5773109243697478, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7789915966386555, "Spanish,Chinese": 0.7260504201680672, "Spanish,Vietnamese": 0.746218487394958, "English,Chinese": 0.7504201680672269, "English,Vietnamese": 0.7579831932773109, "Chinese,Vietnamese": 0.7226890756302521 }, "3_combine": { "Spanish,English,Chinese": 0.6487394957983194, "Spanish,English,Vietnamese": 0.6579831932773109, "Spanish,Chinese,Vietnamese": 0.619327731092437, "English,Chinese,Vietnamese": 0.6386554621848739 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5773109243697478 } }, "AC3_2": 0.7103615837339513, "AC3_3": 0.6586491679837845, "AC3_4": 0.6232366803200022 }, "prompt_4": { "overall_acc": 0.6359243697478991, "language_acc": { "Spanish": 0.6226890756302521, "English": 0.6672268907563025, "Chinese": 0.634453781512605, "Vietnamese": 0.619327731092437 }, "consistency_score_2": 0.7540616246498599, "consistency_score_3": 0.6506302521008404, "consistency_score_4": 0.5907563025210084, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7941176470588235, "Spanish,Chinese": 0.7453781512605042, "Spanish,Vietnamese": 0.7411764705882353, "English,Chinese": 0.7588235294117647, "English,Vietnamese": 0.7571428571428571, "Chinese,Vietnamese": 0.7277310924369748 }, "3_combine": { "Spanish,English,Chinese": 0.6672268907563025, "Spanish,English,Vietnamese": 0.6647058823529411, "Spanish,Chinese,Vietnamese": 0.6285714285714286, "English,Chinese,Vietnamese": 0.6420168067226891 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5907563025210084 } }, "AC3_2": 0.6899726548393105, "AC3_3": 0.643193263538917, "AC3_4": 0.612508760947188 }, "prompt_5": { "overall_acc": 0.6310924369747899, "language_acc": { "Spanish": 0.6176470588235294, "English": 0.6504201680672269, "Chinese": 0.6361344537815126, "Vietnamese": 0.6201680672268908 }, "consistency_score_2": 0.7354341736694678, "consistency_score_3": 0.6235294117647059, "consistency_score_4": 0.5588235294117647, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.7663865546218488, "Spanish,Chinese": 0.7277310924369748, "Spanish,Vietnamese": 0.7294117647058823, "English,Chinese": 0.7470588235294118, "English,Vietnamese": 0.7243697478991596, "Chinese,Vietnamese": 0.7176470588235294 }, "3_combine": { "Spanish,English,Chinese": 0.6403361344537815, "Spanish,English,Vietnamese": 0.6294117647058823, "Spanish,Chinese,Vietnamese": 0.6067226890756302, "English,Chinese,Vietnamese": 0.6176470588235294 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.5588235294117647 } }, "AC3_2": 0.6792797758147215, "AC3_3": 0.6272881288653323, "AC3_4": 0.59276337648724 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.3685714285714285, "language_acc": { "Filipino": 0.29333333333333333, "Vietnamese": 0.38, "Chinese": 0.36666666666666664, "Spanish": 0.3933333333333333, "Malay": 0.35333333333333333, "Indonesian": 0.34, "English": 0.4533333333333333 }, "consistency_score_2": 0.553968253968254, "consistency_score_3": 0.3799999999999999, "consistency_score_4": 0.2857142857142857, "consistency_score_5": 0.22476190476190475, "consistency_score_6": 0.18285714285714286, "consistency_score_7": 0.15333333333333332, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.52, "Filipino,Chinese": 0.47333333333333333, "Filipino,Spanish": 0.5466666666666666, "Filipino,Malay": 0.44, "Filipino,Indonesian": 0.4666666666666667, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.5466666666666666, "Vietnamese,Spanish": 0.5733333333333334, "Vietnamese,Malay": 0.5666666666666667, "Vietnamese,Indonesian": 0.54, "Vietnamese,English": 0.5933333333333334, "Chinese,Spanish": 0.5533333333333333, "Chinese,Malay": 0.56, "Chinese,Indonesian": 0.56, "Chinese,English": 0.58, "Spanish,Malay": 0.5466666666666666, "Spanish,Indonesian": 0.5866666666666667, "Spanish,English": 0.66, "Malay,Indonesian": 0.6533333333333333, "Malay,English": 0.5666666666666667, "Indonesian,English": 0.6066666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.32666666666666666, "Filipino,Vietnamese,Spanish": 0.35333333333333333, "Filipino,Vietnamese,Malay": 0.31333333333333335, "Filipino,Vietnamese,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,English": 0.3333333333333333, "Filipino,Chinese,Spanish": 0.32666666666666666, "Filipino,Chinese,Malay": 0.30666666666666664, "Filipino,Chinese,Indonesian": 0.32, "Filipino,Chinese,English": 0.32666666666666666, "Filipino,Spanish,Malay": 0.3333333333333333, "Filipino,Spanish,Indonesian": 0.36, "Filipino,Spanish,English": 0.38, "Filipino,Malay,Indonesian": 0.3333333333333333, "Filipino,Malay,English": 0.31333333333333335, "Filipino,Indonesian,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish": 0.38, "Vietnamese,Chinese,Malay": 0.38, "Vietnamese,Chinese,Indonesian": 0.38, "Vietnamese,Chinese,English": 0.42, "Vietnamese,Spanish,Malay": 0.4066666666666667, "Vietnamese,Spanish,Indonesian": 0.3933333333333333, "Vietnamese,Spanish,English": 0.4666666666666667, "Vietnamese,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Malay,English": 0.42, "Vietnamese,Indonesian,English": 0.43333333333333335, "Chinese,Spanish,Malay": 0.37333333333333335, "Chinese,Spanish,Indonesian": 0.4, "Chinese,Spanish,English": 0.42, "Chinese,Malay,Indonesian": 0.4266666666666667, "Chinese,Malay,English": 0.3933333333333333, "Chinese,Indonesian,English": 0.4266666666666667, "Spanish,Malay,Indonesian": 0.42, "Spanish,Malay,English": 0.44666666666666666, "Spanish,Indonesian,English": 0.48, "Malay,Indonesian,English": 0.4533333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.23333333333333334, "Filipino,Vietnamese,Chinese,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Chinese,English": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Malay,English": 0.24666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.25333333333333335, "Filipino,Chinese,Spanish,Malay": 0.25333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Chinese,Spanish,English": 0.25333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.26, "Filipino,Chinese,Malay,English": 0.24, "Filipino,Chinese,Indonesian,English": 0.26666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.2733333333333333, "Filipino,Spanish,Malay,English": 0.2733333333333333, "Filipino,Spanish,Indonesian,English": 0.3, "Filipino,Malay,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish,Malay": 0.2866666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Chinese,Spanish,English": 0.32666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Chinese,Malay,English": 0.30666666666666664, "Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.31333333333333335, "Vietnamese,Spanish,Malay,English": 0.36, "Vietnamese,Spanish,Indonesian,English": 0.35333333333333333, "Vietnamese,Malay,Indonesian,English": 0.35333333333333333, "Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, "Chinese,Spanish,Malay,English": 0.31333333333333335, "Chinese,Spanish,Indonesian,English": 0.34, "Chinese,Malay,Indonesian,English": 0.34, "Spanish,Malay,Indonesian,English": 0.38 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Spanish,English": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.21333333333333335, "Filipino,Vietnamese,Malay,Indonesian,English": 0.21333333333333335, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22, "Filipino,Chinese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.22666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.2733333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.3, "Chinese,Spanish,Malay,Indonesian,English": 0.28 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.18, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 } }, "AC3_2": 0.442640841560514, "AC3_3": 0.3741984732324543, "AC3_4": 0.32189644411798835, "AC3_5": 0.2792387066715798, "AC3_6": 0.24444115465589333, "AC3_7": 0.21656934302419742 }, "prompt_2": { "overall_acc": 0.35809523809523813, "language_acc": { "Filipino": 0.30666666666666664, "Vietnamese": 0.37333333333333335, "Chinese": 0.38, "Spanish": 0.38666666666666666, "Malay": 0.30666666666666664, "Indonesian": 0.32666666666666666, "English": 0.4266666666666667 }, "consistency_score_2": 0.5041269841269842, "consistency_score_3": 0.3179047619047619, "consistency_score_4": 0.21771428571428578, "consistency_score_5": 0.15460317460317463, "consistency_score_6": 0.11142857142857143, "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.44666666666666666, "Filipino,Chinese": 0.48, "Filipino,Spanish": 0.4266666666666667, "Filipino,Malay": 0.44666666666666666, "Filipino,Indonesian": 0.4, "Filipino,English": 0.5133333333333333, "Vietnamese,Chinese": 0.5333333333333333, "Vietnamese,Spanish": 0.49333333333333335, "Vietnamese,Malay": 0.5466666666666666, "Vietnamese,Indonesian": 0.4666666666666667, "Vietnamese,English": 0.5666666666666667, "Chinese,Spanish": 0.5133333333333333, "Chinese,Malay": 0.5133333333333333, "Chinese,Indonesian": 0.52, "Chinese,English": 0.6133333333333333, "Spanish,Malay": 0.5266666666666666, "Spanish,Indonesian": 0.52, "Spanish,English": 0.52, "Malay,Indonesian": 0.6266666666666667, "Malay,English": 0.4666666666666667, "Indonesian,English": 0.44666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2866666666666667, "Filipino,Vietnamese,Spanish": 0.26, "Filipino,Vietnamese,Malay": 0.26666666666666666, "Filipino,Vietnamese,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,English": 0.31333333333333335, "Filipino,Chinese,Spanish": 0.26666666666666666, "Filipino,Chinese,Malay": 0.2866666666666667, "Filipino,Chinese,Indonesian": 0.29333333333333333, "Filipino,Chinese,English": 0.3466666666666667, "Filipino,Spanish,Malay": 0.2866666666666667, "Filipino,Spanish,Indonesian": 0.25333333333333335, "Filipino,Spanish,English": 0.26666666666666666, "Filipino,Malay,Indonesian": 0.30666666666666664, "Filipino,Malay,English": 0.2866666666666667, "Filipino,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish": 0.3333333333333333, "Vietnamese,Chinese,Malay": 0.32666666666666666, "Vietnamese,Chinese,Indonesian": 0.3, "Vietnamese,Chinese,English": 0.41333333333333333, "Vietnamese,Spanish,Malay": 0.3466666666666667, "Vietnamese,Spanish,Indonesian": 0.30666666666666664, "Vietnamese,Spanish,English": 0.36666666666666664, "Vietnamese,Malay,Indonesian": 0.37333333333333335, "Vietnamese,Malay,English": 0.3466666666666667, "Vietnamese,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay": 0.3466666666666667, "Chinese,Spanish,Indonesian": 0.34, "Chinese,Spanish,English": 0.36666666666666664, "Chinese,Malay,Indonesian": 0.38, "Chinese,Malay,English": 0.3333333333333333, "Chinese,Indonesian,English": 0.3333333333333333, "Spanish,Malay,Indonesian": 0.3933333333333333, "Spanish,Malay,English": 0.34, "Spanish,Indonesian,English": 0.32666666666666666, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.18, "Filipino,Vietnamese,Chinese,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.16, "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, "Filipino,Vietnamese,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay": 0.2, "Filipino,Chinese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Chinese,Spanish,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Malay,English": 0.2, "Filipino,Chinese,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian": 0.21333333333333335, "Filipino,Spanish,Malay,English": 0.18666666666666668, "Filipino,Spanish,Indonesian,English": 0.18, "Filipino,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Spanish,English": 0.28, "Vietnamese,Chinese,Malay,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Malay,English": 0.24666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.24, "Vietnamese,Spanish,Malay,Indonesian": 0.26, "Vietnamese,Spanish,Malay,English": 0.26, "Vietnamese,Spanish,Indonesian,English": 0.24, "Vietnamese,Malay,Indonesian,English": 0.24, "Chinese,Spanish,Malay,Indonesian": 0.28, "Chinese,Spanish,Malay,English": 0.24, "Chinese,Spanish,Indonesian,English": 0.24, "Chinese,Malay,Indonesian,English": 0.25333333333333335, "Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.14, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.14, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Indonesian,English": 0.14, "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian,English": 0.2, "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, "AC3_2": 0.41874465245160963, "AC3_3": 0.3368052703818208, "AC3_4": 0.27079249558172297, "AC3_5": 0.2159657968029305, "AC3_6": 0.1699681251449068, "AC3_7": 0.1307826086657996 }, "prompt_3": { "overall_acc": 0.3771428571428571, "language_acc": { "Filipino": 0.28, "Vietnamese": 0.35333333333333333, "Chinese": 0.42, "Spanish": 0.38, "Malay": 0.36666666666666664, "Indonesian": 0.36, "English": 0.48 }, "consistency_score_2": 0.5076190476190477, "consistency_score_3": 0.3255238095238095, "consistency_score_4": 0.23104761904761908, "consistency_score_5": 0.17301587301587304, "consistency_score_6": 0.13428571428571429, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4533333333333333, "Filipino,Chinese": 0.47333333333333333, "Filipino,Spanish": 0.4666666666666667, "Filipino,Malay": 0.4266666666666667, "Filipino,Indonesian": 0.42, "Filipino,English": 0.46, "Vietnamese,Chinese": 0.5066666666666667, "Vietnamese,Spanish": 0.44, "Vietnamese,Malay": 0.5333333333333333, "Vietnamese,Indonesian": 0.47333333333333333, "Vietnamese,English": 0.5333333333333333, "Chinese,Spanish": 0.5333333333333333, "Chinese,Malay": 0.5333333333333333, "Chinese,Indonesian": 0.5533333333333333, "Chinese,English": 0.5933333333333334, "Spanish,Malay": 0.56, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.5933333333333334, "Malay,Indonesian": 0.6333333333333333, "Malay,English": 0.5133333333333333, "Indonesian,English": 0.4666666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2733333333333333, "Filipino,Vietnamese,Spanish": 0.26666666666666666, "Filipino,Vietnamese,Malay": 0.26, "Filipino,Vietnamese,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,English": 0.2733333333333333, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.3, "Filipino,Chinese,Indonesian": 0.3, "Filipino,Chinese,English": 0.32, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.26666666666666666, "Filipino,Spanish,English": 0.31333333333333335, "Filipino,Malay,Indonesian": 0.3, "Filipino,Malay,English": 0.29333333333333333, "Filipino,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish": 0.3, "Vietnamese,Chinese,Malay": 0.35333333333333333, "Vietnamese,Chinese,Indonesian": 0.32666666666666666, "Vietnamese,Chinese,English": 0.37333333333333335, "Vietnamese,Spanish,Malay": 0.3333333333333333, "Vietnamese,Spanish,Indonesian": 0.28, "Vietnamese,Spanish,English": 0.35333333333333333, "Vietnamese,Malay,Indonesian": 0.38, "Vietnamese,Malay,English": 0.36666666666666664, "Vietnamese,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay": 0.37333333333333335, "Chinese,Spanish,Indonesian": 0.3466666666666667, "Chinese,Spanish,English": 0.4, "Chinese,Malay,Indonesian": 0.4066666666666667, "Chinese,Malay,English": 0.36, "Chinese,Indonesian,English": 0.35333333333333333, "Spanish,Malay,Indonesian": 0.4, "Spanish,Malay,English": 0.4, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.37333333333333335 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay": 0.22, "Filipino,Chinese,Spanish,Indonesian": 0.20666666666666667, "Filipino,Chinese,Spanish,English": 0.22666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, "Filipino,Chinese,Malay,English": 0.22666666666666666, "Filipino,Chinese,Indonesian,English": 0.22, "Filipino,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Spanish,Malay,English": 0.23333333333333334, "Filipino,Spanish,Indonesian,English": 0.2, "Filipino,Malay,Indonesian,English": 0.22, "Vietnamese,Chinese,Spanish,Malay": 0.24, "Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,Spanish,English": 0.25333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.28, "Vietnamese,Chinese,Malay,English": 0.26, "Vietnamese,Chinese,Indonesian,English": 0.24, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.29333333333333333, "Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, "Vietnamese,Malay,Indonesian,English": 0.26666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.28, "Chinese,Spanish,Malay,English": 0.2866666666666667, "Chinese,Spanish,Indonesian,English": 0.2733333333333333, "Chinese,Malay,Indonesian,English": 0.28, "Spanish,Malay,Indonesian,English": 0.3 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Malay,English": 0.16, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.18, "Filipino,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Malay,Indonesian,English": 0.18, "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, "Vietnamese,Chinese,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.4327602644443982, "AC3_3": 0.3494373232666932, "AC3_4": 0.28654825282747687, "AC3_5": 0.23721045251644143, "AC3_6": 0.19805267354467448, "AC3_7": 0.1662992125640523 }, "prompt_4": { "overall_acc": 0.36476190476190473, "language_acc": { "Filipino": 0.29333333333333333, "Vietnamese": 0.38666666666666666, "Chinese": 0.36666666666666664, "Spanish": 0.3933333333333333, "Malay": 0.36666666666666664, "Indonesian": 0.30666666666666664, "English": 0.44 }, "consistency_score_2": 0.533015873015873, "consistency_score_3": 0.35638095238095235, "consistency_score_4": 0.26342857142857146, "consistency_score_5": 0.2066666666666667, "consistency_score_6": 0.16857142857142857, "consistency_score_7": 0.14, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.43333333333333335, "Filipino,Chinese": 0.5, "Filipino,Spanish": 0.5066666666666667, "Filipino,Malay": 0.4066666666666667, "Filipino,Indonesian": 0.4, "Filipino,English": 0.5, "Vietnamese,Chinese": 0.52, "Vietnamese,Spanish": 0.5466666666666666, "Vietnamese,Malay": 0.5733333333333334, "Vietnamese,Indonesian": 0.54, "Vietnamese,English": 0.5466666666666666, "Chinese,Spanish": 0.58, "Chinese,Malay": 0.54, "Chinese,Indonesian": 0.5266666666666666, "Chinese,English": 0.6066666666666667, "Spanish,Malay": 0.56, "Spanish,Indonesian": 0.5533333333333333, "Spanish,English": 0.66, "Malay,Indonesian": 0.6266666666666667, "Malay,English": 0.54, "Indonesian,English": 0.5266666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.28, "Filipino,Vietnamese,Spanish": 0.29333333333333333, "Filipino,Vietnamese,Malay": 0.28, "Filipino,Vietnamese,Indonesian": 0.26, "Filipino,Vietnamese,English": 0.3, "Filipino,Chinese,Spanish": 0.32666666666666666, "Filipino,Chinese,Malay": 0.2866666666666667, "Filipino,Chinese,Indonesian": 0.29333333333333333, "Filipino,Chinese,English": 0.35333333333333333, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.3, "Filipino,Spanish,English": 0.36666666666666664, "Filipino,Malay,Indonesian": 0.2866666666666667, "Filipino,Malay,English": 0.29333333333333333, "Filipino,Indonesian,English": 0.30666666666666664, "Vietnamese,Chinese,Spanish": 0.37333333333333335, "Vietnamese,Chinese,Malay": 0.36666666666666664, "Vietnamese,Chinese,Indonesian": 0.36, "Vietnamese,Chinese,English": 0.38666666666666666, "Vietnamese,Spanish,Malay": 0.3933333333333333, "Vietnamese,Spanish,Indonesian": 0.38, "Vietnamese,Spanish,English": 0.44, "Vietnamese,Malay,Indonesian": 0.41333333333333333, "Vietnamese,Malay,English": 0.38, "Vietnamese,Indonesian,English": 0.38666666666666666, "Chinese,Spanish,Malay": 0.4, "Chinese,Spanish,Indonesian": 0.38666666666666666, "Chinese,Spanish,English": 0.4533333333333333, "Chinese,Malay,Indonesian": 0.4, "Chinese,Malay,English": 0.38666666666666666, "Chinese,Indonesian,English": 0.38666666666666666, "Spanish,Malay,Indonesian": 0.42, "Spanish,Malay,English": 0.4266666666666667, "Spanish,Indonesian,English": 0.42, "Malay,Indonesian,English": 0.3933333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.22, "Filipino,Vietnamese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.26666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.22666666666666666, "Filipino,Chinese,Malay,English": 0.23333333333333334, "Filipino,Chinese,Indonesian,English": 0.24, "Filipino,Spanish,Malay,Indonesian": 0.22666666666666666, "Filipino,Spanish,Malay,English": 0.24666666666666667, "Filipino,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Chinese,Spanish,English": 0.32, "Vietnamese,Chinese,Malay,Indonesian": 0.29333333333333333, "Vietnamese,Chinese,Malay,English": 0.29333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.30666666666666664, "Vietnamese,Spanish,Malay,Indonesian": 0.30666666666666664, "Vietnamese,Spanish,Malay,English": 0.32666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.32666666666666666, "Vietnamese,Malay,Indonesian,English": 0.3, "Chinese,Spanish,Malay,Indonesian": 0.32, "Chinese,Spanish,Malay,English": 0.32666666666666666, "Chinese,Spanish,Indonesian,English": 0.30666666666666664, "Chinese,Malay,Indonesian,English": 0.31333333333333335, "Spanish,Malay,Indonesian,English": 0.3333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Spanish,English": 0.18, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.2, "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.2, "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay,English": 0.26, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.26, "Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Spanish,Malay,Indonesian,English": 0.26666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14 } }, "AC3_2": 0.43312251628502213, "AC3_3": 0.36052272779444616, "AC3_4": 0.30592220388442065, "AC3_5": 0.2638444443982717, "AC3_6": 0.23058163260982714, "AC3_7": 0.2023396226014233 }, "prompt_5": { "overall_acc": 0.3676190476190477, "language_acc": { "Filipino": 0.2866666666666667, "Vietnamese": 0.38, "Chinese": 0.38, "Spanish": 0.4066666666666667, "Malay": 0.3, "Indonesian": 0.3466666666666667, "English": 0.47333333333333333 }, "consistency_score_2": 0.5006349206349207, "consistency_score_3": 0.3114285714285713, "consistency_score_4": 0.21561904761904763, "consistency_score_5": 0.16031746031746033, "consistency_score_6": 0.12476190476190477, "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.42, "Filipino,Chinese": 0.48, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.4266666666666667, "Filipino,Indonesian": 0.43333333333333335, "Filipino,English": 0.4533333333333333, "Vietnamese,Chinese": 0.52, "Vietnamese,Spanish": 0.49333333333333335, "Vietnamese,Malay": 0.5666666666666667, "Vietnamese,Indonesian": 0.5133333333333333, "Vietnamese,English": 0.54, "Chinese,Spanish": 0.48, "Chinese,Malay": 0.5066666666666667, "Chinese,Indonesian": 0.5, "Chinese,English": 0.58, "Spanish,Malay": 0.4866666666666667, "Spanish,Indonesian": 0.52, "Spanish,English": 0.5666666666666667, "Malay,Indonesian": 0.58, "Malay,English": 0.48, "Indonesian,English": 0.4866666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.26666666666666666, "Filipino,Vietnamese,Spanish": 0.26, "Filipino,Vietnamese,Malay": 0.2733333333333333, "Filipino,Vietnamese,Indonesian": 0.24, "Filipino,Vietnamese,English": 0.26666666666666666, "Filipino,Chinese,Spanish": 0.2733333333333333, "Filipino,Chinese,Malay": 0.26666666666666666, "Filipino,Chinese,Indonesian": 0.28, "Filipino,Chinese,English": 0.30666666666666664, "Filipino,Spanish,Malay": 0.26666666666666666, "Filipino,Spanish,Indonesian": 0.2866666666666667, "Filipino,Spanish,English": 0.3, "Filipino,Malay,Indonesian": 0.2866666666666667, "Filipino,Malay,English": 0.26, "Filipino,Indonesian,English": 0.2733333333333333, "Vietnamese,Chinese,Spanish": 0.31333333333333335, "Vietnamese,Chinese,Malay": 0.3333333333333333, "Vietnamese,Chinese,Indonesian": 0.3333333333333333, "Vietnamese,Chinese,English": 0.38, "Vietnamese,Spanish,Malay": 0.32666666666666666, "Vietnamese,Spanish,Indonesian": 0.32666666666666666, "Vietnamese,Spanish,English": 0.36, "Vietnamese,Malay,Indonesian": 0.38, "Vietnamese,Malay,English": 0.3466666666666667, "Vietnamese,Indonesian,English": 0.3466666666666667, "Chinese,Spanish,Malay": 0.3, "Chinese,Spanish,Indonesian": 0.32, "Chinese,Spanish,English": 0.35333333333333333, "Chinese,Malay,Indonesian": 0.3466666666666667, "Chinese,Malay,English": 0.32666666666666666, "Chinese,Indonesian,English": 0.36, "Spanish,Malay,Indonesian": 0.3466666666666667, "Spanish,Malay,English": 0.32666666666666666, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18, "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,English": 0.2, "Filipino,Vietnamese,Spanish,Malay": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay": 0.18666666666666668, "Filipino,Chinese,Spanish,Indonesian": 0.19333333333333333, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.19333333333333333, "Filipino,Chinese,Malay,English": 0.2, "Filipino,Chinese,Indonesian,English": 0.21333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.22, "Filipino,Spanish,Malay,English": 0.18666666666666668, "Filipino,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.22, "Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Spanish,English": 0.25333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,Malay,English": 0.25333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.2733333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, "Vietnamese,Spanish,Malay,English": 0.23333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Vietnamese,Malay,Indonesian,English": 0.24, "Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, "Chinese,Spanish,Malay,English": 0.23333333333333334, "Chinese,Spanish,Indonesian,English": 0.24666666666666667, "Chinese,Malay,Indonesian,English": 0.25333333333333335, "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14, "Filipino,Vietnamese,Malay,Indonesian,English": 0.14, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Chinese,Spanish,Malay,English": 0.14666666666666667, "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,English": 0.18, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.2, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, "AC3_2": 0.4239380168398303, "AC3_3": 0.33719895807496664, "AC3_4": 0.2718123852602294, "AC3_5": 0.22326833314787048, "AC3_6": 0.1862982407284897, "AC3_7": 0.15723014253256787 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3409090909090909, "language_acc": { "Indonesian": 0.32954545454545453, "English": 0.38636363636363635, "Filipino": 0.2897727272727273, "Spanish": 0.3352272727272727, "Chinese": 0.39204545454545453, "Malay": 0.3693181818181818, "Vietnamese": 0.2840909090909091 }, "consistency_score_2": 0.529491341991342, "consistency_score_3": 0.3487012987012986, "consistency_score_4": 0.2537337662337663, "consistency_score_5": 0.19534632034632035, "consistency_score_6": 0.15503246753246752, "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.625, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.5511363636363636, "Indonesian,Chinese": 0.5, "Indonesian,Malay": 0.6079545454545454, "Indonesian,Vietnamese": 0.5284090909090909, "English,Filipino": 0.4375, "English,Spanish": 0.6534090909090909, "English,Chinese": 0.5965909090909091, "English,Malay": 0.5852272727272727, "English,Vietnamese": 0.5681818181818182, "Filipino,Spanish": 0.4318181818181818, "Filipino,Chinese": 0.4147727272727273, "Filipino,Malay": 0.4659090909090909, "Filipino,Vietnamese": 0.4659090909090909, "Spanish,Chinese": 0.5397727272727273, "Spanish,Malay": 0.5397727272727273, "Spanish,Vietnamese": 0.5397727272727273, "Chinese,Malay": 0.5113636363636364, "Chinese,Vietnamese": 0.5170454545454546, "Malay,Vietnamese": 0.5681818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.30113636363636365, "Indonesian,English,Spanish": 0.45454545454545453, "Indonesian,English,Chinese": 0.4034090909090909, "Indonesian,English,Malay": 0.4375, "Indonesian,English,Vietnamese": 0.4034090909090909, "Indonesian,Filipino,Spanish": 0.2784090909090909, "Indonesian,Filipino,Chinese": 0.25, "Indonesian,Filipino,Malay": 0.32954545454545453, "Indonesian,Filipino,Vietnamese": 0.2897727272727273, "Indonesian,Spanish,Chinese": 0.3465909090909091, "Indonesian,Spanish,Malay": 0.4090909090909091, "Indonesian,Spanish,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay": 0.35795454545454547, "Indonesian,Chinese,Vietnamese": 0.3409090909090909, "Indonesian,Malay,Vietnamese": 0.38636363636363635, "English,Filipino,Spanish": 0.3181818181818182, "English,Filipino,Chinese": 0.2897727272727273, "English,Filipino,Malay": 0.29545454545454547, "English,Filipino,Vietnamese": 0.30113636363636365, "English,Spanish,Chinese": 0.4375, "English,Spanish,Malay": 0.4318181818181818, "English,Spanish,Vietnamese": 0.4431818181818182, "English,Chinese,Malay": 0.39204545454545453, "English,Chinese,Vietnamese": 0.4034090909090909, "English,Malay,Vietnamese": 0.4090909090909091, "Filipino,Spanish,Chinese": 0.26704545454545453, "Filipino,Spanish,Malay": 0.2840909090909091, "Filipino,Spanish,Vietnamese": 0.30113636363636365, "Filipino,Chinese,Malay": 0.25, "Filipino,Chinese,Vietnamese": 0.2840909090909091, "Filipino,Malay,Vietnamese": 0.3068181818181818, "Spanish,Chinese,Malay": 0.3465909090909091, "Spanish,Chinese,Vietnamese": 0.3522727272727273, "Spanish,Malay,Vietnamese": 0.3693181818181818, "Chinese,Malay,Vietnamese": 0.3522727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.22727272727272727, "Indonesian,English,Filipino,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Malay": 0.24431818181818182, "Indonesian,English,Filipino,Vietnamese": 0.2215909090909091, "Indonesian,English,Spanish,Chinese": 0.3181818181818182, "Indonesian,English,Spanish,Malay": 0.3522727272727273, "Indonesian,English,Spanish,Vietnamese": 0.3465909090909091, "Indonesian,English,Chinese,Malay": 0.3125, "Indonesian,English,Chinese,Vietnamese": 0.30113636363636365, "Indonesian,English,Malay,Vietnamese": 0.3181818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Malay": 0.2159090909090909, "Indonesian,Filipino,Spanish,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay": 0.1875, "Indonesian,Filipino,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Chinese,Malay": 0.2784090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.3068181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese": 0.22727272727272727, "English,Filipino,Spanish,Malay": 0.23863636363636365, "English,Filipino,Spanish,Vietnamese": 0.26136363636363635, "English,Filipino,Chinese,Malay": 0.20454545454545456, "English,Filipino,Chinese,Vietnamese": 0.22727272727272727, "English,Filipino,Malay,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese,Malay": 0.3068181818181818, "English,Spanish,Chinese,Vietnamese": 0.32386363636363635, "English,Spanish,Malay,Vietnamese": 0.3352272727272727, "English,Chinese,Malay,Vietnamese": 0.29545454545454547, "Filipino,Spanish,Chinese,Malay": 0.1875, "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.16477272727272727, "Indonesian,English,Filipino,Spanish,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1875, "Indonesian,English,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "English,Filipino,Spanish,Chinese,Vietnamese": 0.1875, "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, "English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "English,Spanish,Chinese,Malay,Vietnamese": 0.25, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, "AC3_2": 0.41477096103975203, "AC3_3": 0.3447611709823638, "AC3_4": 0.2909314272461383, "AC3_5": 0.2483717089712054, "AC3_6": 0.2131379258609069, "AC3_7": 0.18292682922903034 }, "prompt_2": { "overall_acc": 0.31168831168831174, "language_acc": { "Indonesian": 0.29545454545454547, "English": 0.38636363636363635, "Filipino": 0.2556818181818182, "Spanish": 0.2897727272727273, "Chinese": 0.38636363636363635, "Malay": 0.2897727272727273, "Vietnamese": 0.2784090909090909 }, "consistency_score_2": 0.4548160173160174, "consistency_score_3": 0.2663961038961039, "consistency_score_4": 0.175, "consistency_score_5": 0.12337662337662339, "consistency_score_6": 0.09090909090909091, "consistency_score_7": 0.06818181818181818, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5056818181818182, "Indonesian,Filipino": 0.4659090909090909, "Indonesian,Spanish": 0.4034090909090909, "Indonesian,Chinese": 0.42613636363636365, "Indonesian,Malay": 0.5568181818181818, "Indonesian,Vietnamese": 0.44886363636363635, "English,Filipino": 0.4772727272727273, "English,Spanish": 0.5397727272727273, "English,Chinese": 0.6022727272727273, "English,Malay": 0.48863636363636365, "English,Vietnamese": 0.5, "Filipino,Spanish": 0.32386363636363635, "Filipino,Chinese": 0.42613636363636365, "Filipino,Malay": 0.4147727272727273, "Filipino,Vietnamese": 0.42045454545454547, "Spanish,Chinese": 0.4375, "Spanish,Malay": 0.38636363636363635, "Spanish,Vietnamese": 0.42613636363636365, "Chinese,Malay": 0.42045454545454547, "Chinese,Vietnamese": 0.4318181818181818, "Malay,Vietnamese": 0.44886363636363635 }, "3_combine": { "Indonesian,English,Filipino": 0.2897727272727273, "Indonesian,English,Spanish": 0.29545454545454547, "Indonesian,English,Chinese": 0.3181818181818182, "Indonesian,English,Malay": 0.3409090909090909, "Indonesian,English,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Spanish": 0.19886363636363635, "Indonesian,Filipino,Chinese": 0.23863636363636365, "Indonesian,Filipino,Malay": 0.2840909090909091, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.2556818181818182, "Indonesian,Spanish,Malay": 0.2727272727272727, "Indonesian,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,Chinese,Malay": 0.26136363636363635, "Indonesian,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Malay,Vietnamese": 0.3125, "English,Filipino,Spanish": 0.23295454545454544, "English,Filipino,Chinese": 0.3125, "English,Filipino,Malay": 0.2727272727272727, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.3352272727272727, "English,Spanish,Malay": 0.2897727272727273, "English,Spanish,Vietnamese": 0.3125, "English,Chinese,Malay": 0.32386363636363635, "English,Chinese,Vietnamese": 0.32954545454545453, "English,Malay,Vietnamese": 0.29545454545454547, "Filipino,Spanish,Chinese": 0.17613636363636365, "Filipino,Spanish,Malay": 0.18181818181818182, "Filipino,Spanish,Vietnamese": 0.19886363636363635, "Filipino,Chinese,Malay": 0.22727272727272727, "Filipino,Chinese,Vietnamese": 0.22727272727272727, "Filipino,Malay,Vietnamese": 0.23863636363636365, "Spanish,Chinese,Malay": 0.24431818181818182, "Spanish,Chinese,Vietnamese": 0.26136363636363635, "Spanish,Malay,Vietnamese": 0.23863636363636365, "Chinese,Malay,Vietnamese": 0.23863636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.16477272727272727, "Indonesian,English,Filipino,Chinese": 0.18181818181818182, "Indonesian,English,Filipino,Malay": 0.1875, "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Spanish,Malay": 0.2215909090909091, "Indonesian,English,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,English,Chinese,Malay": 0.2215909090909091, "Indonesian,English,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Malay": 0.14204545454545456, "Indonesian,Filipino,Spanish,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.1875, "Indonesian,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Chinese": 0.14772727272727273, "English,Filipino,Spanish,Malay": 0.1590909090909091, "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay": 0.19886363636363635, "English,Filipino,Chinese,Vietnamese": 0.1875, "English,Filipino,Malay,Vietnamese": 0.1875, "English,Spanish,Chinese,Malay": 0.21022727272727273, "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, "English,Spanish,Malay,Vietnamese": 0.20454545454545456, "English,Chinese,Malay,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Chinese,Malay": 0.125, "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Malay": 0.125, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.125, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, "English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 } }, "AC3_2": 0.36988920005626297, "AC3_3": 0.2872679131659877, "AC3_4": 0.22414943290924746, "AC3_5": 0.1767784453945292, "AC3_6": 0.14076246330814493, "AC3_7": 0.11188811185865757 }, "prompt_3": { "overall_acc": 0.3051948051948052, "language_acc": { "Indonesian": 0.2897727272727273, "English": 0.35795454545454547, "Filipino": 0.26704545454545453, "Spanish": 0.2897727272727273, "Chinese": 0.3806818181818182, "Malay": 0.3068181818181818, "Vietnamese": 0.24431818181818182 }, "consistency_score_2": 0.48457792207792205, "consistency_score_3": 0.3008116883116883, "consistency_score_4": 0.20957792207792206, "consistency_score_5": 0.15530303030303033, "consistency_score_6": 0.12012987012987013, "consistency_score_7": 0.09659090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5454545454545454, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.4431818181818182, "Indonesian,Chinese": 0.45454545454545453, "Indonesian,Malay": 0.5965909090909091, "Indonesian,Vietnamese": 0.45454545454545453, "English,Filipino": 0.4602272727272727, "English,Spanish": 0.5965909090909091, "English,Chinese": 0.5909090909090909, "English,Malay": 0.5284090909090909, "English,Vietnamese": 0.5397727272727273, "Filipino,Spanish": 0.3806818181818182, "Filipino,Chinese": 0.4147727272727273, "Filipino,Malay": 0.4318181818181818, "Filipino,Vietnamese": 0.4431818181818182, "Spanish,Chinese": 0.5284090909090909, "Spanish,Malay": 0.4659090909090909, "Spanish,Vietnamese": 0.4431818181818182, "Chinese,Malay": 0.42045454545454547, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.30113636363636365, "Indonesian,English,Spanish": 0.35795454545454547, "Indonesian,English,Chinese": 0.3465909090909091, "Indonesian,English,Malay": 0.39204545454545453, "Indonesian,English,Vietnamese": 0.3352272727272727, "Indonesian,Filipino,Spanish": 0.26136363636363635, "Indonesian,Filipino,Chinese": 0.23295454545454544, "Indonesian,Filipino,Malay": 0.3181818181818182, "Indonesian,Filipino,Vietnamese": 0.26136363636363635, "Indonesian,Spanish,Chinese": 0.2840909090909091, "Indonesian,Spanish,Malay": 0.32954545454545453, "Indonesian,Spanish,Vietnamese": 0.2840909090909091, "Indonesian,Chinese,Malay": 0.3068181818181818, "Indonesian,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,Malay,Vietnamese": 0.32386363636363635, "English,Filipino,Spanish": 0.29545454545454547, "English,Filipino,Chinese": 0.29545454545454547, "English,Filipino,Malay": 0.2840909090909091, "English,Filipino,Vietnamese": 0.30113636363636365, "English,Spanish,Chinese": 0.39204545454545453, "English,Spanish,Malay": 0.3693181818181818, "English,Spanish,Vietnamese": 0.3465909090909091, "English,Chinese,Malay": 0.3409090909090909, "English,Chinese,Vietnamese": 0.3522727272727273, "English,Malay,Vietnamese": 0.3409090909090909, "Filipino,Spanish,Chinese": 0.23863636363636365, "Filipino,Spanish,Malay": 0.24431818181818182, "Filipino,Spanish,Vietnamese": 0.25, "Filipino,Chinese,Malay": 0.2215909090909091, "Filipino,Chinese,Vietnamese": 0.26704545454545453, "Filipino,Malay,Vietnamese": 0.2556818181818182, "Spanish,Chinese,Malay": 0.2840909090909091, "Spanish,Chinese,Vietnamese": 0.30113636363636365, "Spanish,Malay,Vietnamese": 0.30113636363636365, "Chinese,Malay,Vietnamese": 0.2556818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2215909090909091, "Indonesian,English,Filipino,Chinese": 0.1875, "Indonesian,English,Filipino,Malay": 0.22727272727272727, "Indonesian,English,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Chinese": 0.24431818181818182, "Indonesian,English,Spanish,Malay": 0.2840909090909091, "Indonesian,English,Spanish,Vietnamese": 0.25, "Indonesian,English,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,English,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay": 0.16477272727272727, "Indonesian,Filipino,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese,Malay": 0.2215909090909091, "Indonesian,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,Spanish,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Chinese,Malay,Vietnamese": 0.1875, "English,Filipino,Spanish,Chinese": 0.19318181818181818, "English,Filipino,Spanish,Malay": 0.2159090909090909, "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, "English,Filipino,Chinese,Malay": 0.19318181818181818, "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, "English,Filipino,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay": 0.23863636363636365, "English,Spanish,Chinese,Vietnamese": 0.25, "English,Spanish,Malay,Vietnamese": 0.26136363636363635, "English,Chinese,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.13636363636363635, "Indonesian,English,Filipino,Spanish,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2159090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.125, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 } }, "AC3_2": 0.37451448854723246, "AC3_3": 0.3029873956242453, "AC3_4": 0.24850614533582688, "AC3_5": 0.20585407537459033, "AC3_6": 0.17240011892447066, "AC3_7": 0.14674012852178836 }, "prompt_4": { "overall_acc": 0.33035714285714285, "language_acc": { "Indonesian": 0.3125, "English": 0.4034090909090909, "Filipino": 0.26704545454545453, "Spanish": 0.3465909090909091, "Chinese": 0.35795454545454547, "Malay": 0.3352272727272727, "Vietnamese": 0.2897727272727273 }, "consistency_score_2": 0.49783549783549785, "consistency_score_3": 0.31022727272727274, "consistency_score_4": 0.21347402597402593, "consistency_score_5": 0.15449134199134196, "consistency_score_6": 0.11444805194805195, "consistency_score_7": 0.08522727272727272, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5625, "Indonesian,Filipino": 0.48295454545454547, "Indonesian,Spanish": 0.5113636363636364, "Indonesian,Chinese": 0.4602272727272727, "Indonesian,Malay": 0.5681818181818182, "Indonesian,Vietnamese": 0.5113636363636364, "English,Filipino": 0.4375, "English,Spanish": 0.5738636363636364, "English,Chinese": 0.5795454545454546, "English,Malay": 0.5568181818181818, "English,Vietnamese": 0.5681818181818182, "Filipino,Spanish": 0.4147727272727273, "Filipino,Chinese": 0.42045454545454547, "Filipino,Malay": 0.4602272727272727, "Filipino,Vietnamese": 0.44886363636363635, "Spanish,Chinese": 0.4715909090909091, "Spanish,Malay": 0.5056818181818182, "Spanish,Vietnamese": 0.4772727272727273, "Chinese,Malay": 0.44886363636363635, "Chinese,Vietnamese": 0.5, "Malay,Vietnamese": 0.4943181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.29545454545454547, "Indonesian,English,Spanish": 0.3806818181818182, "Indonesian,English,Chinese": 0.36363636363636365, "Indonesian,English,Malay": 0.3806818181818182, "Indonesian,English,Vietnamese": 0.375, "Indonesian,Filipino,Spanish": 0.2784090909090909, "Indonesian,Filipino,Chinese": 0.25, "Indonesian,Filipino,Malay": 0.3125, "Indonesian,Filipino,Vietnamese": 0.2727272727272727, "Indonesian,Spanish,Chinese": 0.3068181818181818, "Indonesian,Spanish,Malay": 0.35795454545454547, "Indonesian,Spanish,Vietnamese": 0.3181818181818182, "Indonesian,Chinese,Malay": 0.30113636363636365, "Indonesian,Chinese,Vietnamese": 0.29545454545454547, "Indonesian,Malay,Vietnamese": 0.3352272727272727, "English,Filipino,Spanish": 0.2897727272727273, "English,Filipino,Chinese": 0.2784090909090909, "English,Filipino,Malay": 0.2840909090909091, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.36363636363636365, "English,Spanish,Malay": 0.38636363636363635, "English,Spanish,Vietnamese": 0.3806818181818182, "English,Chinese,Malay": 0.35795454545454547, "English,Chinese,Vietnamese": 0.3693181818181818, "English,Malay,Vietnamese": 0.3693181818181818, "Filipino,Spanish,Chinese": 0.2215909090909091, "Filipino,Spanish,Malay": 0.2556818181818182, "Filipino,Spanish,Vietnamese": 0.2556818181818182, "Filipino,Chinese,Malay": 0.2159090909090909, "Filipino,Chinese,Vietnamese": 0.26136363636363635, "Filipino,Malay,Vietnamese": 0.26136363636363635, "Spanish,Chinese,Malay": 0.2897727272727273, "Spanish,Chinese,Vietnamese": 0.29545454545454547, "Spanish,Malay,Vietnamese": 0.32386363636363635, "Chinese,Malay,Vietnamese": 0.2840909090909091 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.22727272727272727, "Indonesian,English,Filipino,Chinese": 0.19318181818181818, "Indonesian,English,Filipino,Malay": 0.2215909090909091, "Indonesian,English,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Chinese": 0.26704545454545453, "Indonesian,English,Spanish,Malay": 0.2840909090909091, "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,English,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,English,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.17613636363636365, "Indonesian,Filipino,Spanish,Malay": 0.19318181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.1875, "Indonesian,Filipino,Chinese,Malay": 0.1590909090909091, "Indonesian,Filipino,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese,Malay": 0.23863636363636365, "Indonesian,Spanish,Chinese,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Malay,Vietnamese": 0.25, "Indonesian,Chinese,Malay,Vietnamese": 0.19886363636363635, "English,Filipino,Spanish,Chinese": 0.18181818181818182, "English,Filipino,Spanish,Malay": 0.21022727272727273, "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, "English,Filipino,Chinese,Malay": 0.17613636363636365, "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, "English,Filipino,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay": 0.25, "English,Spanish,Chinese,Vietnamese": 0.25, "English,Spanish,Malay,Vietnamese": 0.2784090909090909, "English,Chinese,Malay,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Chinese,Malay": 0.21022727272727273, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2159090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.125, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, "English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, "English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 } }, "AC3_2": 0.39716245853234056, "AC3_3": 0.3199759249373783, "AC3_4": 0.2593550106132904, "AC3_5": 0.21052893809433348, "AC3_6": 0.17000130340286518, "AC3_7": 0.13549804684239578 }, "prompt_5": { "overall_acc": 0.31980519480519476, "language_acc": { "Indonesian": 0.30113636363636365, "English": 0.38636363636363635, "Filipino": 0.2556818181818182, "Spanish": 0.2727272727272727, "Chinese": 0.39204545454545453, "Malay": 0.38636363636363635, "Vietnamese": 0.24431818181818182 }, "consistency_score_2": 0.4429112554112554, "consistency_score_3": 0.24772727272727277, "consistency_score_4": 0.1517857142857143, "consistency_score_5": 0.0963203463203463, "consistency_score_6": 0.060064935064935064, "consistency_score_7": 0.03409090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5170454545454546, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.375, "Indonesian,Chinese": 0.4431818181818182, "Indonesian,Malay": 0.5227272727272727, "Indonesian,Vietnamese": 0.45454545454545453, "English,Filipino": 0.44886363636363635, "English,Spanish": 0.4715909090909091, "English,Chinese": 0.5965909090909091, "English,Malay": 0.5227272727272727, "English,Vietnamese": 0.48295454545454547, "Filipino,Spanish": 0.2840909090909091, "Filipino,Chinese": 0.42613636363636365, "Filipino,Malay": 0.42613636363636365, "Filipino,Vietnamese": 0.4147727272727273, "Spanish,Chinese": 0.3693181818181818, "Spanish,Malay": 0.3693181818181818, "Spanish,Vietnamese": 0.3806818181818182, "Chinese,Malay": 0.44886363636363635, "Chinese,Vietnamese": 0.4318181818181818, "Malay,Vietnamese": 0.4431818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.2840909090909091, "Indonesian,English,Spanish": 0.26704545454545453, "Indonesian,English,Chinese": 0.32954545454545453, "Indonesian,English,Malay": 0.3522727272727273, "Indonesian,English,Vietnamese": 0.29545454545454547, "Indonesian,Filipino,Spanish": 0.17613636363636365, "Indonesian,Filipino,Chinese": 0.25, "Indonesian,Filipino,Malay": 0.2897727272727273, "Indonesian,Filipino,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese": 0.20454545454545456, "Indonesian,Spanish,Malay": 0.21022727272727273, "Indonesian,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,Chinese,Malay": 0.2784090909090909, "Indonesian,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish": 0.21022727272727273, "English,Filipino,Chinese": 0.29545454545454547, "English,Filipino,Malay": 0.26704545454545453, "English,Filipino,Vietnamese": 0.26136363636363635, "English,Spanish,Chinese": 0.2840909090909091, "English,Spanish,Malay": 0.26136363636363635, "English,Spanish,Vietnamese": 0.2784090909090909, "English,Chinese,Malay": 0.3409090909090909, "English,Chinese,Vietnamese": 0.3125, "English,Malay,Vietnamese": 0.29545454545454547, "Filipino,Spanish,Chinese": 0.14772727272727273, "Filipino,Spanish,Malay": 0.14204545454545456, "Filipino,Spanish,Vietnamese": 0.14204545454545456, "Filipino,Chinese,Malay": 0.22727272727272727, "Filipino,Chinese,Vietnamese": 0.23295454545454544, "Filipino,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay": 0.21022727272727273, "Spanish,Chinese,Vietnamese": 0.20454545454545456, "Spanish,Malay,Vietnamese": 0.21022727272727273, "Chinese,Malay,Vietnamese": 0.23863636363636365 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.13636363636363635, "Indonesian,English,Filipino,Chinese": 0.19318181818181818, "Indonesian,English,Filipino,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, "Indonesian,English,Spanish,Chinese": 0.17045454545454544, "Indonesian,English,Spanish,Malay": 0.17613636363636365, "Indonesian,English,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,English,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Chinese,Vietnamese": 0.1875, "Indonesian,English,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Malay": 0.10227272727272728, "Indonesian,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,Spanish,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese": 0.13068181818181818, "English,Filipino,Spanish,Malay": 0.13068181818181818, "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay": 0.18181818181818182, "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Malay,Vietnamese": 0.17045454545454544, "English,Spanish,Chinese,Malay": 0.18181818181818182, "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, "English,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Chinese,Malay,Vietnamese": 0.19318181818181818, "Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Malay": 0.125, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.125, "Indonesian,English,Spanish,Chinese,Malay": 0.11931818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Chinese,Malay,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "English,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 } }, "AC3_2": 0.37142327337883285, "AC3_3": 0.2791892031960738, "AC3_4": 0.2058642733765583, "AC3_5": 0.14805025919803594, "AC3_6": 0.10113497610835256, "AC3_7": 0.061613844853315454 } }, "sg_eval": { "prompt_1": { "accuracy": 0.33980582524271846 }, "prompt_2": { "accuracy": 0.39805825242718446 }, "prompt_3": { "accuracy": 0.3883495145631068 }, "prompt_4": { "accuracy": 0.3300970873786408 }, "prompt_5": { "accuracy": 0.36893203883495146 } }, "cn_eval": { "prompt_1": { "accuracy": 0.3047619047619048 }, "prompt_2": { "accuracy": 0.3142857142857143 }, "prompt_3": { "accuracy": 0.3238095238095238 }, "prompt_4": { "accuracy": 0.2857142857142857 }, "prompt_5": { "accuracy": 0.2857142857142857 } }, "us_eval": { "prompt_1": { "accuracy": 0.37383177570093457 }, "prompt_2": { "accuracy": 0.34579439252336447 }, "prompt_3": { "accuracy": 0.3364485981308411 }, "prompt_4": { "accuracy": 0.37383177570093457 }, "prompt_5": { "accuracy": 0.37383177570093457 } }, "ph_eval": { "prompt_1": { "accuracy": 0.33, "category_acc": { "brand": 0.4, "demographics": 0.2, "biology": 0.5, "history": 0.13333333333333333, "literature": 0.2, "politics": 0.3, "culture": 0.3, "film": 0.3, "law": 0.2, "geography": 0.8 } }, "prompt_2": { "accuracy": 0.27, "category_acc": { "brand": 0.2, "demographics": 0.0, "biology": 0.5, "history": 0.06666666666666667, "literature": 0.2, "politics": 0.2, "culture": 0.2, "film": 0.3, "law": 0.2, "geography": 0.8 } }, "prompt_3": { "accuracy": 0.28, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.6, "history": 0.06666666666666667, "literature": 0.2, "politics": 0.3, "culture": 0.2, "film": 0.3, "law": 0.2, "geography": 0.6 } }, "prompt_4": { "accuracy": 0.34, "category_acc": { "brand": 0.4, "demographics": 0.2, "biology": 0.5, "history": 0.13333333333333333, "literature": 0.3, "politics": 0.4, "culture": 0.3, "film": 0.3, "law": 0.2, "geography": 0.7 } }, "prompt_5": { "accuracy": 0.29, "category_acc": { "brand": 0.3, "demographics": 0.2, "biology": 0.5, "history": 0.13333333333333333, "literature": 0.2, "politics": 0.2, "culture": 0.3, "film": 0.3, "law": 0.2, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.06761639222925121 }, "prompt_2": { "bleu_score": 0.09051604298209881 }, "prompt_3": { "bleu_score": 0.10102303287817192 }, "prompt_4": { "bleu_score": 0.036990319382943476 }, "prompt_5": { "bleu_score": 0.043004932216465244 } }, "indommlu": { "prompt_1": { "accuracy": 0.3142399359102744, "category_acc": { "History": 0.2751004016064257, "Geography": 0.27346938775510204, "Lampungic": 0.29931972789115646, "Social science": 0.41736227045075125, "Balinese": 0.31634819532908703, "Makassarese": 0.26344086021505375, "Banjarese": 0.2847222222222222, "Chemistry": 0.23503649635036497, "Biology": 0.28402366863905326, "Science": 0.3219814241486068, "Christian religion": 0.32338308457711445, "Art": 0.34442595673876875, "Islam religion": 0.30725462304409673, "Hindu religion": 0.36, "Madurese": 0.2677966101694915, "Sport": 0.3108108108108108, "Indonesian language": 0.3530510585305106, "Physics": 0.24646464646464647, "Minangkabau culture": 0.24120603015075376, "Dayak language": 0.29357798165137616, "Sociology": 0.3064516129032258, "Economy": 0.27049180327868855, "Sundanese": 0.32497839239412274, "Javanese": 0.2762096774193548, "Civic education": 0.36194563662374823 } }, "prompt_2": { "accuracy": 0.32558915815475, "category_acc": { "History": 0.29518072289156627, "Geography": 0.24285714285714285, "Lampungic": 0.3673469387755102, "Social science": 0.4590984974958264, "Balinese": 0.3057324840764331, "Makassarese": 0.2956989247311828, "Banjarese": 0.3402777777777778, "Chemistry": 0.24233576642335766, "Biology": 0.2923076923076923, "Science": 0.34055727554179566, "Christian religion": 0.3582089552238806, "Art": 0.3227953410981697, "Islam religion": 0.33428165007112376, "Hindu religion": 0.35333333333333333, "Madurese": 0.2576271186440678, "Sport": 0.31756756756756754, "Indonesian language": 0.37017434620174344, "Physics": 0.2828282828282828, "Minangkabau culture": 0.2864321608040201, "Dayak language": 0.26605504587155965, "Sociology": 0.3125, "Economy": 0.2807377049180328, "Sundanese": 0.324114088159032, "Javanese": 0.27923387096774194, "Civic education": 0.3648068669527897 } }, "prompt_3": { "accuracy": 0.32392015488350356, "category_acc": { "History": 0.28112449799196787, "Geography": 0.25510204081632654, "Lampungic": 0.3673469387755102, "Social science": 0.4590984974958264, "Balinese": 0.2929936305732484, "Makassarese": 0.27956989247311825, "Banjarese": 0.2916666666666667, "Chemistry": 0.2364963503649635, "Biology": 0.29467455621301775, "Science": 0.3323013415892673, "Christian religion": 0.3482587064676617, "Art": 0.32945091514143093, "Islam religion": 0.33001422475106684, "Hindu religion": 0.36, "Madurese": 0.26101694915254237, "Sport": 0.2905405405405405, "Indonesian language": 0.36924034869240346, "Physics": 0.296969696969697, "Minangkabau culture": 0.2663316582914573, "Dayak language": 0.30275229357798167, "Sociology": 0.30443548387096775, "Economy": 0.26844262295081966, "Sundanese": 0.3267070008643042, "Javanese": 0.2872983870967742, "Civic education": 0.3648068669527897 } }, "prompt_4": { "accuracy": 0.3128379731624274, "category_acc": { "History": 0.28112449799196787, "Geography": 0.25918367346938775, "Lampungic": 0.29931972789115646, "Social science": 0.41903171953255425, "Balinese": 0.3184713375796178, "Makassarese": 0.26344086021505375, "Banjarese": 0.2847222222222222, "Chemistry": 0.23357664233576642, "Biology": 0.2911242603550296, "Science": 0.32094943240454077, "Christian religion": 0.3482587064676617, "Art": 0.3178036605657238, "Islam religion": 0.3058321479374111, "Hindu religion": 0.35333333333333333, "Madurese": 0.26101694915254237, "Sport": 0.33783783783783783, "Indonesian language": 0.34869240348692404, "Physics": 0.26666666666666666, "Minangkabau culture": 0.27638190954773867, "Dayak language": 0.28440366972477066, "Sociology": 0.2963709677419355, "Economy": 0.2827868852459016, "Sundanese": 0.30855661192739847, "Javanese": 0.2842741935483871, "Civic education": 0.3562231759656652 } }, "prompt_5": { "accuracy": 0.3103678483209827, "category_acc": { "History": 0.26907630522088355, "Geography": 0.25918367346938775, "Lampungic": 0.36054421768707484, "Social science": 0.4373956594323873, "Balinese": 0.27176220806794055, "Makassarese": 0.2849462365591398, "Banjarese": 0.2986111111111111, "Chemistry": 0.2364963503649635, "Biology": 0.30414201183431955, "Science": 0.31475748194014447, "Christian religion": 0.3582089552238806, "Art": 0.3427620632279534, "Islam religion": 0.3029871977240398, "Hindu religion": 0.3, "Madurese": 0.2711864406779661, "Sport": 0.3310810810810811, "Indonesian language": 0.34526774595267745, "Physics": 0.2585858585858586, "Minangkabau culture": 0.271356783919598, "Dayak language": 0.28440366972477066, "Sociology": 0.2842741935483871, "Economy": 0.2766393442622951, "Sundanese": 0.2964563526361279, "Javanese": 0.27419354838709675, "Civic education": 0.35336194563662376 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.12263992848849109 }, "prompt_2": { "bleu_score": 0.1405986376170242 }, "prompt_3": { "bleu_score": 0.14780903164607387 }, "prompt_4": { "bleu_score": 0.11593944757406402 }, "prompt_5": { "bleu_score": 0.07257475769504178 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.10048045342384325 }, "prompt_2": { "bleu_score": 0.10932505903021678 }, "prompt_3": { "bleu_score": 0.1152753150757208 }, "prompt_4": { "bleu_score": 0.09481002704233622 }, "prompt_5": { "bleu_score": 0.06238696127281799 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.09529323546043389 }, "prompt_2": { "bleu_score": 0.09823605054446687 }, "prompt_3": { "bleu_score": 0.10268505419891161 }, "prompt_4": { "bleu_score": 0.08875397574829268 }, "prompt_5": { "bleu_score": 0.06588601704918913 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.10372104119194801 }, "prompt_2": { "bleu_score": 0.11353837701704785 }, "prompt_3": { "bleu_score": 0.11657014338539547 }, "prompt_4": { "bleu_score": 0.0899448490064802 }, "prompt_5": { "bleu_score": 0.04398409172523046 } }, "mmlu": { "prompt_1": { "accuracy": 0.4014002333722287 }, "prompt_2": { "accuracy": 0.41423570595099185 }, "prompt_3": { "accuracy": 0.41540256709451573 }, "prompt_4": { "accuracy": 0.40840140023337224 }, "prompt_5": { "accuracy": 0.396732788798133 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.3768323203432249, "category_acc": { "high_school_european_history": 0.4573170731707317, "business_ethics": 0.45454545454545453, "clinical_knowledge": 0.4015151515151515, "medical_genetics": 0.3939393939393939, "high_school_us_history": 0.4088669950738916, "high_school_physics": 0.2733333333333333, "high_school_world_history": 0.4872881355932203, "virology": 0.3696969696969697, "high_school_microeconomics": 0.3291139240506329, "econometrics": 0.23008849557522124, "college_computer_science": 0.2828282828282828, "high_school_biology": 0.43042071197411, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.2846975088967972, "philosophy": 0.36451612903225805, "professional_medicine": 0.31365313653136534, "nutrition": 0.46229508196721314, "global_facts": 0.30303030303030304, "machine_learning": 0.42342342342342343, "security_studies": 0.35655737704918034, "public_relations": 0.44036697247706424, "professional_psychology": 0.38461538461538464, "prehistory": 0.4055727554179567, "anatomy": 0.43283582089552236, "human_sexuality": 0.4153846153846154, "college_medicine": 0.3488372093023256, "high_school_government_and_politics": 0.4322916666666667, "college_chemistry": 0.1919191919191919, "logical_fallacies": 0.4074074074074074, "high_school_geography": 0.467005076142132, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.481981981981982, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.5091911764705882, "formal_logic": 0.344, "high_school_statistics": 0.24186046511627907, "international_law": 0.475, "high_school_mathematics": 0.22676579925650558, "high_school_computer_science": 0.3939393939393939, "conceptual_physics": 0.3504273504273504, "miscellaneous": 0.47698209718670076, "high_school_chemistry": 0.27722772277227725, "marketing": 0.5965665236051502, "professional_law": 0.32093933463796476, "management": 0.43137254901960786, "college_physics": 0.2376237623762376, "jurisprudence": 0.38317757009345793, "world_religions": 0.5117647058823529, "sociology": 0.495, "us_foreign_policy": 0.5252525252525253, "high_school_macroeconomics": 0.37789203084832906, "computer_security": 0.5151515151515151, "moral_scenarios": 0.23937360178970918, "moral_disputes": 0.4608695652173913, "electrical_engineering": 0.4236111111111111, "astronomy": 0.31788079470198677, "college_biology": 0.4125874125874126 } }, "prompt_2": { "accuracy": 0.39234894529853415, "category_acc": { "high_school_european_history": 0.5060975609756098, "business_ethics": 0.5252525252525253, "clinical_knowledge": 0.4128787878787879, "medical_genetics": 0.42424242424242425, "high_school_us_history": 0.3793103448275862, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.5296610169491526, "virology": 0.4121212121212121, "high_school_microeconomics": 0.3291139240506329, "econometrics": 0.25663716814159293, "college_computer_science": 0.32323232323232326, "high_school_biology": 0.45307443365695793, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.28113879003558717, "philosophy": 0.3935483870967742, "professional_medicine": 0.3210332103321033, "nutrition": 0.47540983606557374, "global_facts": 0.3434343434343434, "machine_learning": 0.36936936936936937, "security_studies": 0.4139344262295082, "public_relations": 0.3944954128440367, "professional_psychology": 0.3862520458265139, "prehistory": 0.4241486068111455, "anatomy": 0.41044776119402987, "human_sexuality": 0.4230769230769231, "college_medicine": 0.3953488372093023, "high_school_government_and_politics": 0.484375, "college_chemistry": 0.2828282828282828, "logical_fallacies": 0.4012345679012346, "high_school_geography": 0.48223350253807107, "elementary_mathematics": 0.259946949602122, "human_aging": 0.4594594594594595, "college_mathematics": 0.35353535353535354, "high_school_psychology": 0.5606617647058824, "formal_logic": 0.352, "high_school_statistics": 0.2372093023255814, "international_law": 0.5, "high_school_mathematics": 0.2379182156133829, "high_school_computer_science": 0.3939393939393939, "conceptual_physics": 0.38461538461538464, "miscellaneous": 0.5153452685421995, "high_school_chemistry": 0.30198019801980197, "marketing": 0.5793991416309013, "professional_law": 0.3313763861709067, "management": 0.4411764705882353, "college_physics": 0.2079207920792079, "jurisprudence": 0.4485981308411215, "world_religions": 0.5176470588235295, "sociology": 0.525, "us_foreign_policy": 0.5454545454545454, "high_school_macroeconomics": 0.39588688946015427, "computer_security": 0.5353535353535354, "moral_scenarios": 0.2348993288590604, "moral_disputes": 0.43768115942028984, "electrical_engineering": 0.4583333333333333, "astronomy": 0.3509933774834437, "college_biology": 0.4125874125874126 } }, "prompt_3": { "accuracy": 0.3949231319270647, "category_acc": { "high_school_european_history": 0.5182926829268293, "business_ethics": 0.494949494949495, "clinical_knowledge": 0.4166666666666667, "medical_genetics": 0.43434343434343436, "high_school_us_history": 0.3793103448275862, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.5466101694915254, "virology": 0.4, "high_school_microeconomics": 0.3291139240506329, "econometrics": 0.23008849557522124, "college_computer_science": 0.29292929292929293, "high_school_biology": 0.4563106796116505, "abstract_algebra": 0.24242424242424243, "professional_accounting": 0.28825622775800713, "philosophy": 0.38387096774193546, "professional_medicine": 0.31365313653136534, "nutrition": 0.4819672131147541, "global_facts": 0.32323232323232326, "machine_learning": 0.3783783783783784, "security_studies": 0.4139344262295082, "public_relations": 0.43119266055045874, "professional_psychology": 0.3878887070376432, "prehistory": 0.42105263157894735, "anatomy": 0.41044776119402987, "human_sexuality": 0.4230769230769231, "college_medicine": 0.38372093023255816, "high_school_government_and_politics": 0.46875, "college_chemistry": 0.2727272727272727, "logical_fallacies": 0.3950617283950617, "high_school_geography": 0.48223350253807107, "elementary_mathematics": 0.29442970822281167, "human_aging": 0.46396396396396394, "college_mathematics": 0.3434343434343434, "high_school_psychology": 0.5569852941176471, "formal_logic": 0.352, "high_school_statistics": 0.26046511627906976, "international_law": 0.525, "high_school_mathematics": 0.26022304832713755, "high_school_computer_science": 0.4444444444444444, "conceptual_physics": 0.39316239316239315, "miscellaneous": 0.5191815856777494, "high_school_chemistry": 0.297029702970297, "marketing": 0.5793991416309013, "professional_law": 0.32811480756686234, "management": 0.46078431372549017, "college_physics": 0.2079207920792079, "jurisprudence": 0.42990654205607476, "world_religions": 0.5176470588235295, "sociology": 0.515, "us_foreign_policy": 0.5555555555555556, "high_school_macroeconomics": 0.4138817480719794, "computer_security": 0.5151515151515151, "moral_scenarios": 0.24272930648769575, "moral_disputes": 0.4405797101449275, "electrical_engineering": 0.4444444444444444, "astronomy": 0.3509933774834437, "college_biology": 0.43356643356643354 } }, "prompt_4": { "accuracy": 0.38584197354308186, "category_acc": { "high_school_european_history": 0.47560975609756095, "business_ethics": 0.45454545454545453, "clinical_knowledge": 0.4015151515151515, "medical_genetics": 0.3838383838383838, "high_school_us_history": 0.3842364532019704, "high_school_physics": 0.3, "high_school_world_history": 0.5296610169491526, "virology": 0.37575757575757573, "high_school_microeconomics": 0.32489451476793246, "econometrics": 0.22123893805309736, "college_computer_science": 0.2727272727272727, "high_school_biology": 0.4174757281553398, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.3096085409252669, "philosophy": 0.36451612903225805, "professional_medicine": 0.30996309963099633, "nutrition": 0.46229508196721314, "global_facts": 0.2828282828282828, "machine_learning": 0.36936936936936937, "security_studies": 0.4098360655737705, "public_relations": 0.3944954128440367, "professional_psychology": 0.3862520458265139, "prehistory": 0.4086687306501548, "anatomy": 0.43283582089552236, "human_sexuality": 0.4, "college_medicine": 0.36046511627906974, "high_school_government_and_politics": 0.4270833333333333, "college_chemistry": 0.20202020202020202, "logical_fallacies": 0.41358024691358025, "high_school_geography": 0.49238578680203043, "elementary_mathematics": 0.3076923076923077, "human_aging": 0.5, "college_mathematics": 0.3333333333333333, "high_school_psychology": 0.5091911764705882, "formal_logic": 0.344, "high_school_statistics": 0.2558139534883721, "international_law": 0.49166666666666664, "high_school_mathematics": 0.24907063197026022, "high_school_computer_science": 0.40404040404040403, "conceptual_physics": 0.36324786324786323, "miscellaneous": 0.5, "high_school_chemistry": 0.29207920792079206, "marketing": 0.6094420600858369, "professional_law": 0.3359425962165688, "management": 0.4215686274509804, "college_physics": 0.25742574257425743, "jurisprudence": 0.4766355140186916, "world_religions": 0.5117647058823529, "sociology": 0.475, "us_foreign_policy": 0.5353535353535354, "high_school_macroeconomics": 0.37275064267352187, "computer_security": 0.5454545454545454, "moral_scenarios": 0.24049217002237136, "moral_disputes": 0.4811594202898551, "electrical_engineering": 0.4375, "astronomy": 0.304635761589404, "college_biology": 0.40559440559440557 } }, "prompt_5": { "accuracy": 0.38162316767965676, "category_acc": { "high_school_european_history": 0.4878048780487805, "business_ethics": 0.46464646464646464, "clinical_knowledge": 0.4128787878787879, "medical_genetics": 0.3333333333333333, "high_school_us_history": 0.39408866995073893, "high_school_physics": 0.29333333333333333, "high_school_world_history": 0.5084745762711864, "virology": 0.4, "high_school_microeconomics": 0.3333333333333333, "econometrics": 0.24778761061946902, "college_computer_science": 0.25252525252525254, "high_school_biology": 0.4077669902912621, "abstract_algebra": 0.24242424242424243, "professional_accounting": 0.2846975088967972, "philosophy": 0.3903225806451613, "professional_medicine": 0.3062730627306273, "nutrition": 0.4721311475409836, "global_facts": 0.31313131313131315, "machine_learning": 0.36036036036036034, "security_studies": 0.3975409836065574, "public_relations": 0.4036697247706422, "professional_psychology": 0.3944353518821604, "prehistory": 0.4024767801857585, "anatomy": 0.4253731343283582, "human_sexuality": 0.4230769230769231, "college_medicine": 0.37790697674418605, "high_school_government_and_politics": 0.4270833333333333, "college_chemistry": 0.1919191919191919, "logical_fallacies": 0.4074074074074074, "high_school_geography": 0.47715736040609136, "elementary_mathematics": 0.2864721485411141, "human_aging": 0.481981981981982, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.5055147058823529, "formal_logic": 0.352, "high_school_statistics": 0.2558139534883721, "international_law": 0.49166666666666664, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.3939393939393939, "conceptual_physics": 0.358974358974359, "miscellaneous": 0.48081841432225064, "high_school_chemistry": 0.30198019801980197, "marketing": 0.5536480686695279, "professional_law": 0.33529028049575993, "management": 0.39215686274509803, "college_physics": 0.24752475247524752, "jurisprudence": 0.4392523364485981, "world_religions": 0.5294117647058824, "sociology": 0.48, "us_foreign_policy": 0.5454545454545454, "high_school_macroeconomics": 0.37789203084832906, "computer_security": 0.5454545454545454, "moral_scenarios": 0.22706935123042504, "moral_disputes": 0.45217391304347826, "electrical_engineering": 0.4583333333333333, "astronomy": 0.2980132450331126, "college_biology": 0.38461538461538464 } } }, "c_eval": { "prompt_1": { "accuracy": 0.3424962852897474 }, "prompt_2": { "accuracy": 0.3558692421991085 }, "prompt_3": { "accuracy": 0.3573551263001486 }, "prompt_4": { "accuracy": 0.3209509658246657 }, "prompt_5": { "accuracy": 0.34026745913818723 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.3026151930261519, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.2916666666666667, "computer_architecture": 0.38461538461538464, "college_programming": 0.35714285714285715, "college_physics": 0.2916666666666667, "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.21739130434782608, "high_school_physics": 0.25, "high_school_chemistry": 0.041666666666666664, "high_school_biology": 0.25, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.3333333333333333, "business_administration": 0.2894736842105263, "marxism": 0.5, "mao_zedong_thought": 0.3793103448275862, "education_science": 0.4411764705882353, "teacher_qualification": 0.40816326530612246, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.25, "middle_school_politics": 0.34615384615384615, "middle_school_geography": 0.35294117647058826, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.3333333333333333, "logic": 0.3333333333333333, "law": 0.20689655172413793, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.4, "middle_school_history": 0.2222222222222222, "civil_servant": 0.28846153846153844, "sports_science": 0.2916666666666667, "plant_protection": 0.2962962962962963, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.24074074074074073, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.2962962962962963, "physician": 0.4074074074074074 } }, "prompt_2": { "accuracy": 0.3231631382316314, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.25, "computer_architecture": 0.3076923076923077, "college_programming": 0.30952380952380953, "college_physics": 0.2916666666666667, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.25, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.125, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.041666666666666664, "middle_school_biology": 0.4230769230769231, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.36666666666666664, "business_administration": 0.34210526315789475, "marxism": 0.5, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.5, "teacher_qualification": 0.42857142857142855, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.46153846153846156, "middle_school_geography": 0.47058823529411764, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.37037037037037035, "law": 0.2413793103448276, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.44, "middle_school_history": 0.25925925925925924, "civil_servant": 0.28846153846153844, "sports_science": 0.25, "plant_protection": 0.2222222222222222, "basic_medicine": 0.25, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.37254901960784315, "accountant": 0.2962962962962963, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.37037037037037035, "physician": 0.42592592592592593 } }, "prompt_3": { "accuracy": 0.3275217932752179, "category_acc": { "computer_network": 0.3333333333333333, "operating_system": 0.25, "computer_architecture": 0.34615384615384615, "college_programming": 0.3333333333333333, "college_physics": 0.3333333333333333, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.30434782608695654, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2857142857142857, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.125, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.125, "middle_school_biology": 0.4230769230769231, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.35, "business_administration": 0.34210526315789475, "marxism": 0.5, "mao_zedong_thought": 0.41379310344827586, "education_science": 0.5, "teacher_qualification": 0.46938775510204084, "high_school_politics": 0.25, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.46153846153846156, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.4642857142857143, "ideological_and_moral_cultivation": 0.20833333333333334, "logic": 0.37037037037037035, "law": 0.27586206896551724, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.3684210526315789, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.25, "high_school_history": 0.4, "middle_school_history": 0.25925925925925924, "civil_servant": 0.28846153846153844, "sports_science": 0.25, "plant_protection": 0.2222222222222222, "basic_medicine": 0.25, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.2962962962962963, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.4074074074074074, "physician": 0.4074074074074074 } }, "prompt_4": { "accuracy": 0.30074719800747196, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.25, "computer_architecture": 0.3076923076923077, "college_programming": 0.35714285714285715, "college_physics": 0.3333333333333333, "college_chemistry": 0.1724137931034483, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.3333333333333333, "metrology_engineer": 0.3793103448275862, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.125, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.16666666666666666, "middle_school_biology": 0.38461538461538464, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.2, "veterinary_medicine": 0.2857142857142857, "college_economics": 0.2833333333333333, "business_administration": 0.34210526315789475, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.3793103448275862, "education_science": 0.4411764705882353, "teacher_qualification": 0.42857142857142855, "high_school_politics": 0.25, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.4230769230769231, "middle_school_geography": 0.29411764705882354, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.375, "logic": 0.48148148148148145, "law": 0.2413793103448276, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.17857142857142858, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.44, "middle_school_history": 0.2962962962962963, "civil_servant": 0.28846153846153844, "sports_science": 0.2916666666666667, "plant_protection": 0.2222222222222222, "basic_medicine": 0.25, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.2037037037037037, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.25, "tax_accountant": 0.2962962962962963, "physician": 0.4074074074074074 } }, "prompt_5": { "accuracy": 0.30323785803237857, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.2916666666666667, "computer_architecture": 0.3076923076923077, "college_programming": 0.3333333333333333, "college_physics": 0.2916666666666667, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.375, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.08333333333333333, "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.125, "middle_school_biology": 0.3076923076923077, "middle_school_physics": 0.5, "middle_school_chemistry": 0.24, "veterinary_medicine": 0.32142857142857145, "college_economics": 0.2833333333333333, "business_administration": 0.3157894736842105, "marxism": 0.5, "mao_zedong_thought": 0.3793103448275862, "education_science": 0.5, "teacher_qualification": 0.40816326530612246, "high_school_politics": 0.25, "high_school_geography": 0.2916666666666667, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.375, "logic": 0.3333333333333333, "law": 0.20689655172413793, "chinese_language_and_literature": 0.25, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.2647058823529412, "legal_professional": 0.17857142857142858, "high_school_chinese": 0.20833333333333334, "high_school_history": 0.4, "middle_school_history": 0.3333333333333333, "civil_servant": 0.25, "sports_science": 0.2916666666666667, "plant_protection": 0.2222222222222222, "basic_medicine": 0.2916666666666667, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.29411764705882354, "accountant": 0.2777777777777778, "fire_engineer": 0.19444444444444445, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.3148148148148148, "physician": 0.37037037037037035 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.3118279569892473 }, "prompt_2": { "accuracy": 0.3010752688172043 }, "prompt_3": { "accuracy": 0.3225806451612903 }, "prompt_4": { "accuracy": 0.3010752688172043 }, "prompt_5": { "accuracy": 0.3046594982078853 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.32343291314108097, "category_acc": { "agronomy": 0.30177514792899407, "anatomy": 0.2972972972972973, "ancient_chinese": 0.27439024390243905, "arts": 0.3125, "astronomy": 0.3151515151515151, "business_ethics": 0.3588516746411483, "chinese_civil_service_exam": 0.3375, "chinese_driving_rule": 0.45038167938931295, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.3177570093457944, "chinese_history": 0.30959752321981426, "chinese_literature": 0.3088235294117647, "chinese_teacher_qualification": 0.33519553072625696, "clinical_knowledge": 0.3037974683544304, "college_actuarial_science": 0.24528301886792453, "college_education": 0.40186915887850466, "college_engineering_hydrology": 0.3490566037735849, "college_law": 0.25925925925925924, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.3113207547169811, "college_medicine": 0.28205128205128205, "computer_science": 0.37745098039215685, "computer_security": 0.34502923976608185, "conceptual_physics": 0.29931972789115646, "construction_project_management": 0.3669064748201439, "economics": 0.33962264150943394, "education": 0.3496932515337423, "electrical_engineering": 0.3546511627906977, "elementary_chinese": 0.29365079365079366, "elementary_commonsense": 0.31313131313131315, "elementary_information_and_technology": 0.42436974789915966, "elementary_mathematics": 0.25217391304347825, "ethnology": 0.28888888888888886, "food_science": 0.35664335664335667, "genetics": 0.3068181818181818, "global_facts": 0.28187919463087246, "high_school_biology": 0.30177514792899407, "high_school_chemistry": 0.25, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.2636363636363636, "high_school_politics": 0.3356643356643357, "human_sexuality": 0.3253968253968254, "international_law": 0.3027027027027027, "journalism": 0.3546511627906977, "jurisprudence": 0.35036496350364965, "legal_and_moral_basis": 0.46261682242990654, "logical": 0.2764227642276423, "machine_learning": 0.22131147540983606, "management": 0.38095238095238093, "marketing": 0.40555555555555556, "marxist_theory": 0.35978835978835977, "modern_chinese": 0.1896551724137931, "nutrition": 0.36551724137931035, "philosophy": 0.34285714285714286, "professional_accounting": 0.3314285714285714, "professional_law": 0.3175355450236967, "professional_medicine": 0.26063829787234044, "professional_psychology": 0.3448275862068966, "public_relations": 0.3735632183908046, "security_study": 0.34814814814814815, "sociology": 0.3584070796460177, "sports_science": 0.3515151515151515, "traditional_chinese_medicine": 0.2864864864864865, "virology": 0.3254437869822485, "world_history": 0.34782608695652173, "world_religions": 0.30625 } }, "prompt_2": { "accuracy": 0.33474356760490415, "category_acc": { "agronomy": 0.33136094674556216, "anatomy": 0.27702702702702703, "ancient_chinese": 0.25609756097560976, "arts": 0.3375, "astronomy": 0.3212121212121212, "business_ethics": 0.36363636363636365, "chinese_civil_service_exam": 0.325, "chinese_driving_rule": 0.48854961832061067, "chinese_food_culture": 0.20588235294117646, "chinese_foreign_policy": 0.35514018691588783, "chinese_history": 0.33746130030959753, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.3743016759776536, "clinical_knowledge": 0.2742616033755274, "college_actuarial_science": 0.2830188679245283, "college_education": 0.38317757009345793, "college_engineering_hydrology": 0.3867924528301887, "college_law": 0.25925925925925924, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.33962264150943394, "college_medicine": 0.315018315018315, "computer_science": 0.3284313725490196, "computer_security": 0.32748538011695905, "conceptual_physics": 0.3129251700680272, "construction_project_management": 0.37410071942446044, "economics": 0.3522012578616352, "education": 0.4171779141104294, "electrical_engineering": 0.36046511627906974, "elementary_chinese": 0.29365079365079366, "elementary_commonsense": 0.35353535353535354, "elementary_information_and_technology": 0.44537815126050423, "elementary_mathematics": 0.25217391304347825, "ethnology": 0.28888888888888886, "food_science": 0.3776223776223776, "genetics": 0.32386363636363635, "global_facts": 0.26174496644295303, "high_school_biology": 0.2781065088757396, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.3050847457627119, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.3356643356643357, "human_sexuality": 0.3888888888888889, "international_law": 0.2810810810810811, "journalism": 0.36046511627906974, "jurisprudence": 0.36009732360097324, "legal_and_moral_basis": 0.5467289719626168, "logical": 0.2845528455284553, "machine_learning": 0.20491803278688525, "management": 0.3476190476190476, "marketing": 0.4, "marxist_theory": 0.37037037037037035, "modern_chinese": 0.22413793103448276, "nutrition": 0.36551724137931035, "philosophy": 0.38095238095238093, "professional_accounting": 0.32571428571428573, "professional_law": 0.32701421800947866, "professional_medicine": 0.2553191489361702, "professional_psychology": 0.3448275862068966, "public_relations": 0.3793103448275862, "security_study": 0.37037037037037035, "sociology": 0.36283185840707965, "sports_science": 0.4121212121212121, "traditional_chinese_medicine": 0.3081081081081081, "virology": 0.38461538461538464, "world_history": 0.36645962732919257, "world_religions": 0.35 } }, "prompt_3": { "accuracy": 0.3345708858573649, "category_acc": { "agronomy": 0.33136094674556216, "anatomy": 0.28378378378378377, "ancient_chinese": 0.2682926829268293, "arts": 0.3375, "astronomy": 0.3090909090909091, "business_ethics": 0.37320574162679426, "chinese_civil_service_exam": 0.30625, "chinese_driving_rule": 0.48091603053435117, "chinese_food_culture": 0.25, "chinese_foreign_policy": 0.34579439252336447, "chinese_history": 0.3281733746130031, "chinese_literature": 0.3088235294117647, "chinese_teacher_qualification": 0.3854748603351955, "clinical_knowledge": 0.27848101265822783, "college_actuarial_science": 0.2830188679245283, "college_education": 0.40186915887850466, "college_engineering_hydrology": 0.37735849056603776, "college_law": 0.28703703703703703, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.33962264150943394, "college_medicine": 0.2967032967032967, "computer_science": 0.3284313725490196, "computer_security": 0.3333333333333333, "conceptual_physics": 0.3197278911564626, "construction_project_management": 0.38848920863309355, "economics": 0.34591194968553457, "education": 0.4049079754601227, "electrical_engineering": 0.37209302325581395, "elementary_chinese": 0.2857142857142857, "elementary_commonsense": 0.3333333333333333, "elementary_information_and_technology": 0.4831932773109244, "elementary_mathematics": 0.2391304347826087, "ethnology": 0.25925925925925924, "food_science": 0.34965034965034963, "genetics": 0.32954545454545453, "global_facts": 0.2550335570469799, "high_school_biology": 0.28402366863905326, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.3135593220338983, "high_school_mathematics": 0.23780487804878048, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.34965034965034963, "human_sexuality": 0.38095238095238093, "international_law": 0.2864864864864865, "journalism": 0.36627906976744184, "jurisprudence": 0.3746958637469586, "legal_and_moral_basis": 0.5420560747663551, "logical": 0.3008130081300813, "machine_learning": 0.21311475409836064, "management": 0.35714285714285715, "marketing": 0.37222222222222223, "marxist_theory": 0.37037037037037035, "modern_chinese": 0.2413793103448276, "nutrition": 0.36551724137931035, "philosophy": 0.3142857142857143, "professional_accounting": 0.3485714285714286, "professional_law": 0.32701421800947866, "professional_medicine": 0.26595744680851063, "professional_psychology": 0.34051724137931033, "public_relations": 0.40804597701149425, "security_study": 0.32592592592592595, "sociology": 0.37168141592920356, "sports_science": 0.4121212121212121, "traditional_chinese_medicine": 0.31891891891891894, "virology": 0.3727810650887574, "world_history": 0.37267080745341613, "world_religions": 0.33125 } }, "prompt_4": { "accuracy": 0.31402175790018994, "category_acc": { "agronomy": 0.3076923076923077, "anatomy": 0.3108108108108108, "ancient_chinese": 0.2865853658536585, "arts": 0.3375, "astronomy": 0.3151515151515151, "business_ethics": 0.3157894736842105, "chinese_civil_service_exam": 0.3125, "chinese_driving_rule": 0.37404580152671757, "chinese_food_culture": 0.22058823529411764, "chinese_foreign_policy": 0.32710280373831774, "chinese_history": 0.30959752321981426, "chinese_literature": 0.2647058823529412, "chinese_teacher_qualification": 0.35195530726256985, "clinical_knowledge": 0.28270042194092826, "college_actuarial_science": 0.22641509433962265, "college_education": 0.411214953271028, "college_engineering_hydrology": 0.33962264150943394, "college_law": 0.25925925925925924, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.330188679245283, "college_medicine": 0.2564102564102564, "computer_science": 0.3431372549019608, "computer_security": 0.3216374269005848, "conceptual_physics": 0.2857142857142857, "construction_project_management": 0.33093525179856115, "economics": 0.33962264150943394, "education": 0.3558282208588957, "electrical_engineering": 0.3372093023255814, "elementary_chinese": 0.2777777777777778, "elementary_commonsense": 0.31313131313131315, "elementary_information_and_technology": 0.40756302521008403, "elementary_mathematics": 0.26521739130434785, "ethnology": 0.3333333333333333, "food_science": 0.3006993006993007, "genetics": 0.3068181818181818, "global_facts": 0.24161073825503357, "high_school_biology": 0.25443786982248523, "high_school_chemistry": 0.23484848484848486, "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.32167832167832167, "human_sexuality": 0.30952380952380953, "international_law": 0.31891891891891894, "journalism": 0.32558139534883723, "jurisprudence": 0.34306569343065696, "legal_and_moral_basis": 0.4485981308411215, "logical": 0.3170731707317073, "machine_learning": 0.20491803278688525, "management": 0.3619047619047619, "marketing": 0.38333333333333336, "marxist_theory": 0.328042328042328, "modern_chinese": 0.25, "nutrition": 0.3103448275862069, "philosophy": 0.37142857142857144, "professional_accounting": 0.32, "professional_law": 0.2843601895734597, "professional_medicine": 0.26595744680851063, "professional_psychology": 0.3620689655172414, "public_relations": 0.3735632183908046, "security_study": 0.3333333333333333, "sociology": 0.336283185840708, "sports_science": 0.30303030303030304, "traditional_chinese_medicine": 0.2756756756756757, "virology": 0.3076923076923077, "world_history": 0.35403726708074534, "world_religions": 0.34375 } }, "prompt_5": { "accuracy": 0.328268002072181, "category_acc": { "agronomy": 0.3136094674556213, "anatomy": 0.2972972972972973, "ancient_chinese": 0.2865853658536585, "arts": 0.3125, "astronomy": 0.34545454545454546, "business_ethics": 0.36363636363636365, "chinese_civil_service_exam": 0.2875, "chinese_driving_rule": 0.4122137404580153, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.3364485981308411, "chinese_history": 0.3188854489164087, "chinese_literature": 0.3235294117647059, "chinese_teacher_qualification": 0.35195530726256985, "clinical_knowledge": 0.28270042194092826, "college_actuarial_science": 0.22641509433962265, "college_education": 0.3925233644859813, "college_engineering_hydrology": 0.3490566037735849, "college_law": 0.26851851851851855, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.37735849056603776, "college_medicine": 0.315018315018315, "computer_science": 0.35294117647058826, "computer_security": 0.36257309941520466, "conceptual_physics": 0.29931972789115646, "construction_project_management": 0.37410071942446044, "economics": 0.33962264150943394, "education": 0.3803680981595092, "electrical_engineering": 0.32558139534883723, "elementary_chinese": 0.30158730158730157, "elementary_commonsense": 0.30303030303030304, "elementary_information_and_technology": 0.42016806722689076, "elementary_mathematics": 0.2608695652173913, "ethnology": 0.3037037037037037, "food_science": 0.3356643356643357, "genetics": 0.3181818181818182, "global_facts": 0.2550335570469799, "high_school_biology": 0.31952662721893493, "high_school_chemistry": 0.26515151515151514, "high_school_geography": 0.288135593220339, "high_school_mathematics": 0.2804878048780488, "high_school_physics": 0.3, "high_school_politics": 0.3356643356643357, "human_sexuality": 0.3492063492063492, "international_law": 0.2972972972972973, "journalism": 0.3488372093023256, "jurisprudence": 0.35279805352798055, "legal_and_moral_basis": 0.4672897196261682, "logical": 0.2926829268292683, "machine_learning": 0.2540983606557377, "management": 0.3619047619047619, "marketing": 0.4166666666666667, "marxist_theory": 0.3439153439153439, "modern_chinese": 0.21551724137931033, "nutrition": 0.3448275862068966, "philosophy": 0.34285714285714286, "professional_accounting": 0.33714285714285713, "professional_law": 0.3175355450236967, "professional_medicine": 0.2712765957446808, "professional_psychology": 0.34051724137931033, "public_relations": 0.3793103448275862, "security_study": 0.35555555555555557, "sociology": 0.35398230088495575, "sports_science": 0.3939393939393939, "traditional_chinese_medicine": 0.2972972972972973, "virology": 0.3431952662721893, "world_history": 0.36024844720496896, "world_religions": 0.33125 } } }, "zbench": { "prompt_1": { "accuracy": 0.18181818181818182 }, "prompt_2": { "accuracy": 0.24242424242424243 }, "prompt_3": { "accuracy": 0.2727272727272727 }, "prompt_4": { "accuracy": 0.15151515151515152 }, "prompt_5": { "accuracy": 0.18181818181818182 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.18863636363636363 }, "prompt_2": { "accuracy": 0.3477272727272727 }, "prompt_3": { "accuracy": 0.32727272727272727 }, "prompt_4": { "accuracy": 0.3159090909090909 }, "prompt_5": { "accuracy": 0.29772727272727273 } }, "ocnli": { "prompt_1": { "accuracy": 0.3183050847457627 }, "prompt_2": { "accuracy": 0.3338983050847458 }, "prompt_3": { "accuracy": 0.3311864406779661 }, "prompt_4": { "accuracy": 0.32949152542372884 }, "prompt_5": { "accuracy": 0.32949152542372884 } }, "c3": { "prompt_1": { "accuracy": 0.5272999252056844 }, "prompt_2": { "accuracy": 0.5736724008975318 }, "prompt_3": { "accuracy": 0.5762902019446522 }, "prompt_4": { "accuracy": 0.537023186237846 }, "prompt_5": { "accuracy": 0.5340314136125655 } }, "dream": { "prompt_1": { "accuracy": 0.6408623223909848 }, "prompt_2": { "accuracy": 0.6389024987751103 }, "prompt_3": { "accuracy": 0.6242038216560509 }, "prompt_4": { "accuracy": 0.6124448799608035 }, "prompt_5": { "accuracy": 0.636452719255267 } }, "samsum": { "prompt_1": { "rouge1": 0.3119529263953155, "rouge2": 0.0985892127463842, "rougeL": 0.22776684666349456, "avg_rouge": 0.21276966193506475 }, "prompt_2": { "rouge1": 0.31417729846526016, "rouge2": 0.10174884908657739, "rougeL": 0.2284364446741544, "avg_rouge": 0.2147875307419973 }, "prompt_3": { "rouge1": 0.3197747504433708, "rouge2": 0.10071565219435796, "rougeL": 0.2302979113628945, "avg_rouge": 0.21692943800020772 }, "prompt_4": { "rouge1": 0.3144914902100538, "rouge2": 0.10484992173459623, "rougeL": 0.23138751148167905, "avg_rouge": 0.21690964114210967 }, "prompt_5": { "rouge1": 0.32478626515207437, "rouge2": 0.1060837758547477, "rougeL": 0.2387211187380636, "avg_rouge": 0.22319705324829522 } }, "dialogsum": { "prompt_1": { "rouge1": 0.22830350772851404, "rouge2": 0.05711664405435237, "rougeL": 0.17011048466797438, "avg_rouge": 0.1518435454836136 }, "prompt_2": { "rouge1": 0.23975589624985819, "rouge2": 0.05768403212847386, "rougeL": 0.18170612624260932, "avg_rouge": 0.1597153515403138 }, "prompt_3": { "rouge1": 0.23479967755059908, "rouge2": 0.05581131133227013, "rougeL": 0.17651361233518814, "avg_rouge": 0.1557082004060191 }, "prompt_4": { "rouge1": 0.18715038624080982, "rouge2": 0.04242155868919753, "rougeL": 0.1456168620909003, "avg_rouge": 0.12506293567363588 }, "prompt_5": { "rouge1": 0.231174254019962, "rouge2": 0.05141656735407145, "rougeL": 0.17718216315628108, "avg_rouge": 0.15325766151010486 } }, "sst2": { "prompt_1": { "accuracy": 0.8107798165137615 }, "prompt_2": { "accuracy": 0.819954128440367 }, "prompt_3": { "accuracy": 0.7855504587155964 }, "prompt_4": { "accuracy": 0.7855504587155964 }, "prompt_5": { "accuracy": 0.8451834862385321 } }, "cola": { "prompt_1": { "accuracy": 0.47651006711409394 }, "prompt_2": { "accuracy": 0.5302013422818792 }, "prompt_3": { "accuracy": 0.6289549376797698 }, "prompt_4": { "accuracy": 0.6442953020134228 }, "prompt_5": { "accuracy": 0.5953978906999041 } }, "qqp": { "prompt_1": { "accuracy": 0.3935 }, "prompt_2": { "accuracy": 0.4575 }, "prompt_3": { "accuracy": 0.407 }, "prompt_4": { "accuracy": 0.388 }, "prompt_5": { "accuracy": 0.44 } }, "mnli": { "prompt_1": { "accuracy": 0.3 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.3 } }, "qnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.6 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.6 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.6 } }, "rte": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.7 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.7 } }, "mrpc": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.4 }, "prompt_5": { "accuracy": 0.4 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "gemma_7b_it": { "model_size": "7B", "model_link": "https://huggingface.co/google/gemma-7b-it", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.3941176470588235, "language_acc": { "Spanish": 0.4134453781512605, "English": 0.40672268907563025, "Chinese": 0.3689075630252101, "Vietnamese": 0.3873949579831933 }, "consistency_score_2": 0.3878151260504202, "consistency_score_3": 0.22668067226890756, "consistency_score_4": 0.17647058823529413, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.40336134453781514, "Spanish,Chinese": 0.37899159663865545, "Spanish,Vietnamese": 0.3831932773109244, "English,Chinese": 0.37478991596638656, "English,Vietnamese": 0.40672268907563025, "Chinese,Vietnamese": 0.3798319327731092 }, "3_combine": { "Spanish,English,Chinese": 0.22436974789915967, "Spanish,English,Vietnamese": 0.24033613445378152, "Spanish,Chinese,Vietnamese": 0.21596638655462186, "English,Chinese,Vietnamese": 0.22605042016806723 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.17647058823529413 } }, "AC3_2": 0.3909409867692339, "AC3_3": 0.28781924948085924, "AC3_4": 0.2437841115400524 }, "prompt_2": { "overall_acc": 0.3918067226890756, "language_acc": { "Spanish": 0.380672268907563, "English": 0.42521008403361343, "Chinese": 0.3840336134453782, "Vietnamese": 0.3773109243697479 }, "consistency_score_2": 0.3872549019607843, "consistency_score_3": 0.22668067226890756, "consistency_score_4": 0.17899159663865546, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.3957983193277311, "Spanish,Chinese": 0.39747899159663863, "Spanish,Vietnamese": 0.3831932773109244, "English,Chinese": 0.3899159663865546, "English,Vietnamese": 0.3680672268907563, "Chinese,Vietnamese": 0.38907563025210085 }, "3_combine": { "Spanish,English,Chinese": 0.23529411764705882, "Spanish,English,Vietnamese": 0.22773109243697479, "Spanish,Chinese,Vietnamese": 0.22605042016806723, "English,Chinese,Vietnamese": 0.21764705882352942 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.17899159663865546 } }, "AC3_2": 0.3895175148211187, "AC3_3": 0.28720071583774376, "AC3_4": 0.2457264097700475 }, "prompt_3": { "overall_acc": 0.382563025210084, "language_acc": { "Spanish": 0.4092436974789916, "English": 0.3957983193277311, "Chinese": 0.35126050420168065, "Vietnamese": 0.3739495798319328 }, "consistency_score_2": 0.3848739495798319, "consistency_score_3": 0.2197478991596639, "consistency_score_4": 0.17058823529411765, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.41092436974789914, "Spanish,Chinese": 0.3798319327731092, "Spanish,Vietnamese": 0.37478991596638656, "English,Chinese": 0.3941176470588235, "English,Vietnamese": 0.3722689075630252, "Chinese,Vietnamese": 0.3773109243697479 }, "3_combine": { "Spanish,English,Chinese": 0.22773109243697479, "Spanish,English,Vietnamese": 0.22521008403361345, "Spanish,Chinese,Vietnamese": 0.21596638655462186, "English,Chinese,Vietnamese": 0.21008403361344538 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.17058823529411765 } }, "AC3_2": 0.38371500798989855, "AC3_3": 0.27914958095801207, "AC3_4": 0.23595987574206592 }, "prompt_4": { "overall_acc": 0.38193277310924373, "language_acc": { "Spanish": 0.3840336134453782, "English": 0.4092436974789916, "Chinese": 0.35714285714285715, "Vietnamese": 0.3773109243697479 }, "consistency_score_2": 0.39481792717086833, "consistency_score_3": 0.2361344537815126, "consistency_score_4": 0.1865546218487395, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.3941176470588235, "Spanish,Chinese": 0.4042016806722689, "Spanish,Vietnamese": 0.3915966386554622, "English,Chinese": 0.3899159663865546, "English,Vietnamese": 0.4, "Chinese,Vietnamese": 0.38907563025210085 }, "3_combine": { "Spanish,English,Chinese": 0.246218487394958, "Spanish,English,Vietnamese": 0.23445378151260504, "Spanish,Chinese,Vietnamese": 0.23277310924369748, "English,Chinese,Vietnamese": 0.23109243697478993 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.1865546218487395 } }, "AC3_2": 0.3882684771930567, "AC3_3": 0.29183714269567146, "AC3_4": 0.25066984660853586 }, "prompt_5": { "overall_acc": 0.3928571428571429, "language_acc": { "Spanish": 0.3915966386554622, "English": 0.39915966386554624, "Chinese": 0.3815126050420168, "Vietnamese": 0.39915966386554624 }, "consistency_score_2": 0.38837535014005603, "consistency_score_3": 0.22584033613445376, "consistency_score_4": 0.17478991596638654, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.40336134453781514, "Spanish,Chinese": 0.39663865546218485, "Spanish,Vietnamese": 0.3739495798319328, "English,Chinese": 0.3831932773109244, "English,Vietnamese": 0.3840336134453782, "Chinese,Vietnamese": 0.38907563025210085 }, "3_combine": { "Spanish,English,Chinese": 0.22605042016806723, "Spanish,English,Vietnamese": 0.2218487394957983, "Spanish,Chinese,Vietnamese": 0.226890756302521, "English,Chinese,Vietnamese": 0.22857142857142856 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.17478991596638654 } }, "AC3_2": 0.3906033908223061, "AC3_3": 0.28680572394103143, "AC3_4": 0.2419371893409044 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.4333333333333333, "language_acc": { "Filipino": 0.43333333333333335, "Vietnamese": 0.3933333333333333, "Chinese": 0.3933333333333333, "Spanish": 0.43333333333333335, "Malay": 0.44, "Indonesian": 0.38666666666666666, "English": 0.5533333333333333 }, "consistency_score_2": 0.4565079365079366, "consistency_score_3": 0.2859047619047619, "consistency_score_4": 0.21485714285714286, "consistency_score_5": 0.17714285714285713, "consistency_score_6": 0.15238095238095242, "consistency_score_7": 0.13333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.5333333333333333, "Filipino,Chinese": 0.4066666666666667, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.4533333333333333, "Filipino,Indonesian": 0.5133333333333333, "Filipino,English": 0.49333333333333335, "Vietnamese,Chinese": 0.37333333333333335, "Vietnamese,Spanish": 0.44666666666666666, "Vietnamese,Malay": 0.4533333333333333, "Vietnamese,Indonesian": 0.4666666666666667, "Vietnamese,English": 0.4666666666666667, "Chinese,Spanish": 0.4066666666666667, "Chinese,Malay": 0.3466666666666667, "Chinese,Indonesian": 0.4, "Chinese,English": 0.36, "Spanish,Malay": 0.47333333333333333, "Spanish,Indonesian": 0.5, "Spanish,English": 0.5666666666666667, "Malay,Indonesian": 0.5466666666666666, "Malay,English": 0.43333333333333335, "Indonesian,English": 0.4666666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.28, "Filipino,Vietnamese,Spanish": 0.32, "Filipino,Vietnamese,Malay": 0.3333333333333333, "Filipino,Vietnamese,Indonesian": 0.34, "Filipino,Vietnamese,English": 0.32, "Filipino,Chinese,Spanish": 0.26, "Filipino,Chinese,Malay": 0.23333333333333334, "Filipino,Chinese,Indonesian": 0.25333333333333335, "Filipino,Chinese,English": 0.24666666666666667, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.32666666666666666, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.3466666666666667, "Filipino,Malay,English": 0.3, "Filipino,Indonesian,English": 0.32, "Vietnamese,Chinese,Spanish": 0.22, "Vietnamese,Chinese,Malay": 0.22, "Vietnamese,Chinese,Indonesian": 0.25333333333333335, "Vietnamese,Chinese,English": 0.22, "Vietnamese,Spanish,Malay": 0.3, "Vietnamese,Spanish,Indonesian": 0.32, "Vietnamese,Spanish,English": 0.32666666666666666, "Vietnamese,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Malay,English": 0.26666666666666666, "Vietnamese,Indonesian,English": 0.3, "Chinese,Spanish,Malay": 0.22, "Chinese,Spanish,Indonesian": 0.26666666666666666, "Chinese,Spanish,English": 0.26666666666666666, "Chinese,Malay,Indonesian": 0.22, "Chinese,Malay,English": 0.20666666666666667, "Chinese,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian": 0.3466666666666667, "Spanish,Malay,English": 0.31333333333333335, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.2, "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,Spanish,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.25333333333333335, "Filipino,Chinese,Spanish,Malay": 0.18666666666666668, "Filipino,Chinese,Spanish,Indonesian": 0.2, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Malay,English": 0.17333333333333334, "Filipino,Chinese,Indonesian,English": 0.18666666666666668, "Filipino,Spanish,Malay,Indonesian": 0.26, "Filipino,Spanish,Malay,English": 0.24666666666666667, "Filipino,Spanish,Indonesian,English": 0.25333333333333335, "Filipino,Malay,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Spanish,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian": 0.17333333333333334, "Vietnamese,Chinese,Malay,English": 0.16, "Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,Malay,English": 0.22666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.26, "Vietnamese,Malay,Indonesian,English": 0.24, "Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Chinese,Spanish,Malay,English": 0.18, "Chinese,Spanish,Indonesian,English": 0.2, "Chinese,Malay,Indonesian,English": 0.16, "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,Spanish,English": 0.18, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Malay,English": 0.2, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22, "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Chinese,Spanish,Malay,English": 0.16, "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, "Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 } }, "AC3_2": 0.4446188606944744, "AC3_3": 0.3445091807430612, "AC3_4": 0.28727593295597414, "AC3_5": 0.2514820592411769, "AC3_6": 0.22547425470405183, "AC3_7": 0.2039215685914648 }, "prompt_2": { "overall_acc": 0.4247619047619048, "language_acc": { "Filipino": 0.4, "Vietnamese": 0.38666666666666666, "Chinese": 0.4, "Spanish": 0.43333333333333335, "Malay": 0.38, "Indonesian": 0.43333333333333335, "English": 0.54 }, "consistency_score_2": 0.4600000000000001, "consistency_score_3": 0.28819047619047616, "consistency_score_4": 0.21580952380952384, "consistency_score_5": 0.17746031746031748, "consistency_score_6": 0.15238095238095237, "consistency_score_7": 0.13333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4533333333333333, "Filipino,Chinese": 0.3933333333333333, "Filipino,Spanish": 0.5333333333333333, "Filipino,Malay": 0.47333333333333333, "Filipino,Indonesian": 0.4666666666666667, "Filipino,English": 0.4666666666666667, "Vietnamese,Chinese": 0.38666666666666666, "Vietnamese,Spanish": 0.47333333333333333, "Vietnamese,Malay": 0.44, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.48, "Chinese,Spanish": 0.44, "Chinese,Malay": 0.4066666666666667, "Chinese,Indonesian": 0.44, "Chinese,English": 0.42, "Spanish,Malay": 0.49333333333333335, "Spanish,Indonesian": 0.47333333333333333, "Spanish,English": 0.5266666666666666, "Malay,Indonesian": 0.4866666666666667, "Malay,English": 0.4666666666666667, "Indonesian,English": 0.44666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.24666666666666667, "Filipino,Vietnamese,Spanish": 0.32, "Filipino,Vietnamese,Malay": 0.29333333333333333, "Filipino,Vietnamese,Indonesian": 0.31333333333333335, "Filipino,Vietnamese,English": 0.29333333333333333, "Filipino,Chinese,Spanish": 0.2733333333333333, "Filipino,Chinese,Malay": 0.23333333333333334, "Filipino,Chinese,Indonesian": 0.26, "Filipino,Chinese,English": 0.25333333333333335, "Filipino,Spanish,Malay": 0.34, "Filipino,Spanish,Indonesian": 0.32, "Filipino,Spanish,English": 0.35333333333333333, "Filipino,Malay,Indonesian": 0.32, "Filipino,Malay,English": 0.30666666666666664, "Filipino,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish": 0.24666666666666667, "Vietnamese,Chinese,Malay": 0.24666666666666667, "Vietnamese,Chinese,Indonesian": 0.24666666666666667, "Vietnamese,Chinese,English": 0.25333333333333335, "Vietnamese,Spanish,Malay": 0.29333333333333333, "Vietnamese,Spanish,Indonesian": 0.32666666666666666, "Vietnamese,Spanish,English": 0.32, "Vietnamese,Malay,Indonesian": 0.3, "Vietnamese,Malay,English": 0.29333333333333333, "Vietnamese,Indonesian,English": 0.30666666666666664, "Chinese,Spanish,Malay": 0.25333333333333335, "Chinese,Spanish,Indonesian": 0.26, "Chinese,Spanish,English": 0.2733333333333333, "Chinese,Malay,Indonesian": 0.26, "Chinese,Malay,English": 0.25333333333333335, "Chinese,Indonesian,English": 0.26666666666666666, "Spanish,Malay,Indonesian": 0.31333333333333335, "Spanish,Malay,English": 0.34, "Spanish,Indonesian,English": 0.32, "Malay,Indonesian,English": 0.29333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay": 0.18, "Filipino,Vietnamese,Chinese,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Chinese,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.24, "Filipino,Vietnamese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay": 0.19333333333333333, "Filipino,Chinese,Spanish,Indonesian": 0.2, "Filipino,Chinese,Spanish,English": 0.22, "Filipino,Chinese,Malay,Indonesian": 0.19333333333333333, "Filipino,Chinese,Malay,English": 0.18, "Filipino,Chinese,Indonesian,English": 0.19333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Spanish,Malay,English": 0.2733333333333333, "Filipino,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Spanish,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Malay,English": 0.2, "Vietnamese,Chinese,Indonesian,English": 0.19333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.24666666666666667, "Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay,Indonesian": 0.18, "Chinese,Spanish,Malay,English": 0.2, "Chinese,Spanish,Indonesian,English": 0.20666666666666667, "Chinese,Malay,Indonesian,English": 0.18666666666666668, "Spanish,Malay,Indonesian,English": 0.25333333333333335 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Malay,English": 0.16, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, "Vietnamese,Chinese,Spanish,Malay,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, "Vietnamese,Chinese,Malay,Indonesian,English": 0.16, "Vietnamese,Spanish,Malay,Indonesian,English": 0.22, "Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 } }, "AC3_2": 0.4416792249231687, "AC3_3": 0.3433955446511464, "AC3_4": 0.2862059102111859, "AC3_5": 0.25033411146281204, "AC3_6": 0.224296715346961, "AC3_7": 0.20295790667580674 }, "prompt_3": { "overall_acc": 0.41809523809523813, "language_acc": { "Filipino": 0.37333333333333335, "Vietnamese": 0.35333333333333333, "Chinese": 0.4066666666666667, "Spanish": 0.44666666666666666, "Malay": 0.36666666666666664, "Indonesian": 0.44, "English": 0.54 }, "consistency_score_2": 0.46, "consistency_score_3": 0.28571428571428564, "consistency_score_4": 0.20780952380952378, "consistency_score_5": 0.16444444444444442, "consistency_score_6": 0.13523809523809524, "consistency_score_7": 0.11333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.46, "Filipino,Chinese": 0.42, "Filipino,Spanish": 0.5, "Filipino,Malay": 0.44666666666666666, "Filipino,Indonesian": 0.46, "Filipino,English": 0.4533333333333333, "Vietnamese,Chinese": 0.3466666666666667, "Vietnamese,Spanish": 0.46, "Vietnamese,Malay": 0.5133333333333333, "Vietnamese,Indonesian": 0.47333333333333333, "Vietnamese,English": 0.48, "Chinese,Spanish": 0.4266666666666667, "Chinese,Malay": 0.34, "Chinese,Indonesian": 0.44, "Chinese,English": 0.38, "Spanish,Malay": 0.5, "Spanish,Indonesian": 0.4866666666666667, "Spanish,English": 0.58, "Malay,Indonesian": 0.5133333333333333, "Malay,English": 0.5133333333333333, "Indonesian,English": 0.4666666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.22, "Filipino,Vietnamese,Spanish": 0.3, "Filipino,Vietnamese,Malay": 0.3, "Filipino,Vietnamese,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,English": 0.2866666666666667, "Filipino,Chinese,Spanish": 0.25333333333333335, "Filipino,Chinese,Malay": 0.2, "Filipino,Chinese,Indonesian": 0.23333333333333334, "Filipino,Chinese,English": 0.24, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.36, "Filipino,Malay,Indonesian": 0.29333333333333333, "Filipino,Malay,English": 0.30666666666666664, "Filipino,Indonesian,English": 0.2866666666666667, "Vietnamese,Chinese,Spanish": 0.24, "Vietnamese,Chinese,Malay": 0.22, "Vietnamese,Chinese,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,English": 0.23333333333333334, "Vietnamese,Spanish,Malay": 0.32666666666666666, "Vietnamese,Spanish,Indonesian": 0.31333333333333335, "Vietnamese,Spanish,English": 0.34, "Vietnamese,Malay,Indonesian": 0.32666666666666666, "Vietnamese,Malay,English": 0.32666666666666666, "Vietnamese,Indonesian,English": 0.29333333333333333, "Chinese,Spanish,Malay": 0.23333333333333334, "Chinese,Spanish,Indonesian": 0.28, "Chinese,Spanish,English": 0.2866666666666667, "Chinese,Malay,Indonesian": 0.24666666666666667, "Chinese,Malay,English": 0.24666666666666667, "Chinese,Indonesian,English": 0.26666666666666666, "Spanish,Malay,Indonesian": 0.3466666666666667, "Spanish,Malay,English": 0.37333333333333335, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,English": 0.24, "Filipino,Vietnamese,Malay,Indonesian": 0.22, "Filipino,Vietnamese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay": 0.16, "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Chinese,Spanish,English": 0.2, "Filipino,Chinese,Malay,Indonesian": 0.15333333333333332, "Filipino,Chinese,Malay,English": 0.16666666666666666, "Filipino,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, "Filipino,Spanish,Malay,English": 0.26666666666666666, "Filipino,Spanish,Indonesian,English": 0.24666666666666667, "Filipino,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Spanish,English": 0.20666666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Malay,English": 0.18, "Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,Malay,English": 0.26, "Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, "Vietnamese,Malay,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Chinese,Spanish,Malay,English": 0.20666666666666667, "Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Chinese,Malay,Indonesian,English": 0.19333333333333333, "Spanish,Malay,Indonesian,English": 0.28 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,English": 0.16, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.14, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.2, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Malay,English": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.16, "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 } }, "AC3_2": 0.43804772229284705, "AC3_3": 0.3394548617341002, "AC3_4": 0.27762745264091754, "AC3_5": 0.2360472297505786, "AC3_6": 0.20437013355867256, "AC3_7": 0.17832735958412596 }, "prompt_4": { "overall_acc": 0.4247619047619048, "language_acc": { "Filipino": 0.41333333333333333, "Vietnamese": 0.36666666666666664, "Chinese": 0.4066666666666667, "Spanish": 0.43333333333333335, "Malay": 0.3933333333333333, "Indonesian": 0.41333333333333333, "English": 0.5466666666666666 }, "consistency_score_2": 0.4695238095238094, "consistency_score_3": 0.29657142857142865, "consistency_score_4": 0.22209523809523812, "consistency_score_5": 0.1815873015873016, "consistency_score_6": 0.1542857142857143, "consistency_score_7": 0.13333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.48, "Filipino,Chinese": 0.41333333333333333, "Filipino,Spanish": 0.4866666666666667, "Filipino,Malay": 0.46, "Filipino,Indonesian": 0.4266666666666667, "Filipino,English": 0.4533333333333333, "Vietnamese,Chinese": 0.38666666666666666, "Vietnamese,Spanish": 0.5066666666666667, "Vietnamese,Malay": 0.4666666666666667, "Vietnamese,Indonesian": 0.5, "Vietnamese,English": 0.44666666666666666, "Chinese,Spanish": 0.49333333333333335, "Chinese,Malay": 0.36666666666666664, "Chinese,Indonesian": 0.44666666666666666, "Chinese,English": 0.44666666666666666, "Spanish,Malay": 0.48, "Spanish,Indonesian": 0.4866666666666667, "Spanish,English": 0.5666666666666667, "Malay,Indonesian": 0.5266666666666666, "Malay,English": 0.5133333333333333, "Indonesian,English": 0.5066666666666667 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.25333333333333335, "Filipino,Vietnamese,Spanish": 0.32, "Filipino,Vietnamese,Malay": 0.29333333333333333, "Filipino,Vietnamese,Indonesian": 0.30666666666666664, "Filipino,Vietnamese,English": 0.3, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.20666666666666667, "Filipino,Chinese,Indonesian": 0.26, "Filipino,Chinese,English": 0.26666666666666666, "Filipino,Spanish,Malay": 0.30666666666666664, "Filipino,Spanish,Indonesian": 0.30666666666666664, "Filipino,Spanish,English": 0.34, "Filipino,Malay,Indonesian": 0.30666666666666664, "Filipino,Malay,English": 0.30666666666666664, "Filipino,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish": 0.2866666666666667, "Vietnamese,Chinese,Malay": 0.23333333333333334, "Vietnamese,Chinese,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,English": 0.26666666666666666, "Vietnamese,Spanish,Malay": 0.29333333333333333, "Vietnamese,Spanish,Indonesian": 0.3466666666666667, "Vietnamese,Spanish,English": 0.3333333333333333, "Vietnamese,Malay,Indonesian": 0.32, "Vietnamese,Malay,English": 0.28, "Vietnamese,Indonesian,English": 0.32, "Chinese,Spanish,Malay": 0.25333333333333335, "Chinese,Spanish,Indonesian": 0.3, "Chinese,Spanish,English": 0.3333333333333333, "Chinese,Malay,Indonesian": 0.26666666666666666, "Chinese,Malay,English": 0.26666666666666666, "Chinese,Indonesian,English": 0.29333333333333333, "Spanish,Malay,Indonesian": 0.32666666666666666, "Spanish,Malay,English": 0.3466666666666667, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.3466666666666667 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.16, "Filipino,Vietnamese,Chinese,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,English": 0.21333333333333335, "Filipino,Vietnamese,Spanish,Malay": 0.22, "Filipino,Vietnamese,Spanish,Indonesian": 0.24, "Filipino,Vietnamese,Spanish,English": 0.24, "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Malay,English": 0.22, "Filipino,Vietnamese,Indonesian,English": 0.22666666666666666, "Filipino,Chinese,Spanish,Malay": 0.18, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.23333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.18, "Filipino,Chinese,Malay,English": 0.18, "Filipino,Chinese,Indonesian,English": 0.21333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.24, "Filipino,Spanish,Malay,English": 0.24666666666666667, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Spanish,English": 0.23333333333333334, "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Malay,English": 0.19333333333333333, "Vietnamese,Chinese,Indonesian,English": 0.21333333333333335, "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Vietnamese,Spanish,Malay,English": 0.23333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.26, "Vietnamese,Malay,Indonesian,English": 0.24666666666666667, "Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, "Chinese,Spanish,Malay,English": 0.23333333333333334, "Chinese,Spanish,Indonesian,English": 0.24666666666666667, "Chinese,Malay,Indonesian,English": 0.22, "Spanish,Malay,Indonesian,English": 0.26 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18, "Filipino,Vietnamese,Chinese,Spanish,English": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Vietnamese,Malay,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, "Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, "Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 } }, "AC3_2": 0.4460226177299315, "AC3_3": 0.3492760948627414, "AC3_4": 0.29167984739447295, "AC3_5": 0.25441236595155975, "AC3_6": 0.22635338341955605, "AC3_7": 0.20295790667580674 }, "prompt_5": { "overall_acc": 0.42666666666666664, "language_acc": { "Filipino": 0.4, "Vietnamese": 0.38666666666666666, "Chinese": 0.4066666666666667, "Spanish": 0.41333333333333333, "Malay": 0.4266666666666667, "Indonesian": 0.4, "English": 0.5533333333333333 }, "consistency_score_2": 0.46571428571428575, "consistency_score_3": 0.2918095238095238, "consistency_score_4": 0.21733333333333335, "consistency_score_5": 0.17650793650793653, "consistency_score_6": 0.14857142857142858, "consistency_score_7": 0.12666666666666668, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.46, "Filipino,Chinese": 0.38, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.4666666666666667, "Filipino,Indonesian": 0.4666666666666667, "Filipino,English": 0.4533333333333333, "Vietnamese,Chinese": 0.42, "Vietnamese,Spanish": 0.5133333333333333, "Vietnamese,Malay": 0.4533333333333333, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.4266666666666667, "Chinese,Spanish": 0.47333333333333333, "Chinese,Malay": 0.4, "Chinese,Indonesian": 0.52, "Chinese,English": 0.47333333333333333, "Spanish,Malay": 0.47333333333333333, "Spanish,Indonesian": 0.5333333333333333, "Spanish,English": 0.5, "Malay,Indonesian": 0.5066666666666667, "Malay,English": 0.4666666666666667, "Indonesian,English": 0.44666666666666666 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.25333333333333335, "Filipino,Vietnamese,Spanish": 0.30666666666666664, "Filipino,Vietnamese,Malay": 0.3, "Filipino,Vietnamese,Indonesian": 0.32, "Filipino,Vietnamese,English": 0.26666666666666666, "Filipino,Chinese,Spanish": 0.25333333333333335, "Filipino,Chinese,Malay": 0.23333333333333334, "Filipino,Chinese,Indonesian": 0.26, "Filipino,Chinese,English": 0.26, "Filipino,Spanish,Malay": 0.2733333333333333, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.3, "Filipino,Malay,Indonesian": 0.30666666666666664, "Filipino,Malay,English": 0.30666666666666664, "Filipino,Indonesian,English": 0.2866666666666667, "Vietnamese,Chinese,Spanish": 0.2866666666666667, "Vietnamese,Chinese,Malay": 0.25333333333333335, "Vietnamese,Chinese,Indonesian": 0.3, "Vietnamese,Chinese,English": 0.28, "Vietnamese,Spanish,Malay": 0.2733333333333333, "Vietnamese,Spanish,Indonesian": 0.35333333333333333, "Vietnamese,Spanish,English": 0.3, "Vietnamese,Malay,Indonesian": 0.32, "Vietnamese,Malay,English": 0.2733333333333333, "Vietnamese,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay": 0.26666666666666666, "Chinese,Spanish,Indonesian": 0.32, "Chinese,Spanish,English": 0.3, "Chinese,Malay,Indonesian": 0.2866666666666667, "Chinese,Malay,English": 0.28, "Chinese,Indonesian,English": 0.30666666666666664, "Spanish,Malay,Indonesian": 0.32666666666666666, "Spanish,Malay,English": 0.31333333333333335, "Spanish,Indonesian,English": 0.32666666666666666, "Malay,Indonesian,English": 0.32 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.2, "Filipino,Vietnamese,Chinese,Malay": 0.18, "Filipino,Vietnamese,Chinese,Indonesian": 0.2, "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian": 0.24, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Malay,English": 0.22666666666666666, "Filipino,Vietnamese,Indonesian,English": 0.22, "Filipino,Chinese,Spanish,Malay": 0.18, "Filipino,Chinese,Spanish,Indonesian": 0.2, "Filipino,Chinese,Spanish,English": 0.21333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.18666666666666668, "Filipino,Chinese,Malay,English": 0.2, "Filipino,Chinese,Indonesian,English": 0.21333333333333335, "Filipino,Spanish,Malay,Indonesian": 0.22, "Filipino,Spanish,Malay,English": 0.24, "Filipino,Spanish,Indonesian,English": 0.22666666666666666, "Filipino,Malay,Indonesian,English": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, "Vietnamese,Chinese,Spanish,English": 0.22666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.22, "Vietnamese,Chinese,Malay,English": 0.20666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.22, "Vietnamese,Spanish,Malay,Indonesian": 0.24, "Vietnamese,Spanish,Malay,English": 0.21333333333333335, "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, "Vietnamese,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.22, "Chinese,Spanish,Malay,English": 0.21333333333333335, "Chinese,Spanish,Indonesian,English": 0.23333333333333334, "Chinese,Malay,Indonesian,English": 0.22666666666666666, "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, "Filipino,Chinese,Spanish,Indonesian,English": 0.18, "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,English": 0.18, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 } }, "AC3_2": 0.44533617924572005, "AC3_3": 0.34658183098746315, "AC3_4": 0.28797791575928566, "AC3_5": 0.24971228066035472, "AC3_6": 0.22039735095506338, "AC3_7": 0.19534136542654473 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.3717532467532467, "language_acc": { "Indonesian": 0.32386363636363635, "English": 0.42045454545454547, "Filipino": 0.29545454545454547, "Spanish": 0.4147727272727273, "Chinese": 0.3409090909090909, "Malay": 0.4147727272727273, "Vietnamese": 0.39204545454545453 }, "consistency_score_2": 0.4656385281385281, "consistency_score_3": 0.2813311688311689, "consistency_score_4": 0.20129870129870134, "consistency_score_5": 0.15827922077922077, "consistency_score_6": 0.1314935064935065, "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.42045454545454547, "Indonesian,Filipino": 0.4090909090909091, "Indonesian,Spanish": 0.48863636363636365, "Indonesian,Chinese": 0.3522727272727273, "Indonesian,Malay": 0.5625, "Indonesian,Vietnamese": 0.4943181818181818, "English,Filipino": 0.4090909090909091, "English,Spanish": 0.6136363636363636, "English,Chinese": 0.4034090909090909, "English,Malay": 0.48295454545454547, "English,Vietnamese": 0.5625, "Filipino,Spanish": 0.4375, "Filipino,Chinese": 0.36363636363636365, "Filipino,Malay": 0.4772727272727273, "Filipino,Vietnamese": 0.45454545454545453, "Spanish,Chinese": 0.4318181818181818, "Spanish,Malay": 0.5284090909090909, "Spanish,Vietnamese": 0.5454545454545454, "Chinese,Malay": 0.35795454545454547, "Chinese,Vietnamese": 0.4318181818181818, "Malay,Vietnamese": 0.5511363636363636 }, "3_combine": { "Indonesian,English,Filipino": 0.23863636363636365, "Indonesian,English,Spanish": 0.3352272727272727, "Indonesian,English,Chinese": 0.20454545454545456, "Indonesian,English,Malay": 0.30113636363636365, "Indonesian,English,Vietnamese": 0.3068181818181818, "Indonesian,Filipino,Spanish": 0.26704545454545453, "Indonesian,Filipino,Chinese": 0.1875, "Indonesian,Filipino,Malay": 0.2840909090909091, "Indonesian,Filipino,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Chinese": 0.23863636363636365, "Indonesian,Spanish,Malay": 0.35795454545454547, "Indonesian,Spanish,Vietnamese": 0.3409090909090909, "Indonesian,Chinese,Malay": 0.23863636363636365, "Indonesian,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,Malay,Vietnamese": 0.3693181818181818, "English,Filipino,Spanish": 0.2897727272727273, "English,Filipino,Chinese": 0.19318181818181818, "English,Filipino,Malay": 0.2727272727272727, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.29545454545454547, "English,Spanish,Malay": 0.375, "English,Spanish,Vietnamese": 0.4147727272727273, "English,Chinese,Malay": 0.22727272727272727, "English,Chinese,Vietnamese": 0.2784090909090909, "English,Malay,Vietnamese": 0.3522727272727273, "Filipino,Spanish,Chinese": 0.2215909090909091, "Filipino,Spanish,Malay": 0.30113636363636365, "Filipino,Spanish,Vietnamese": 0.2840909090909091, "Filipino,Chinese,Malay": 0.2159090909090909, "Filipino,Chinese,Vietnamese": 0.2215909090909091, "Filipino,Malay,Vietnamese": 0.3125, "Spanish,Chinese,Malay": 0.2556818181818182, "Spanish,Chinese,Vietnamese": 0.26704545454545453, "Spanish,Malay,Vietnamese": 0.375, "Chinese,Malay,Vietnamese": 0.25 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.2159090909090909, "Indonesian,English,Filipino,Chinese": 0.14204545454545456, "Indonesian,English,Filipino,Malay": 0.20454545454545456, "Indonesian,English,Filipino,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Chinese": 0.1875, "Indonesian,English,Spanish,Malay": 0.2784090909090909, "Indonesian,English,Spanish,Vietnamese": 0.26136363636363635, "Indonesian,English,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,English,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Malay": 0.22727272727272727, "Indonesian,Filipino,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Chinese,Malay": 0.1534090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese,Malay": 0.19318181818181818, "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.17613636363636365, "English,Filipino,Spanish,Chinese": 0.17045454545454544, "English,Filipino,Spanish,Malay": 0.23863636363636365, "English,Filipino,Spanish,Vietnamese": 0.24431818181818182, "English,Filipino,Chinese,Malay": 0.1534090909090909, "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Malay,Vietnamese": 0.23295454545454544, "English,Spanish,Chinese,Malay": 0.19318181818181818, "English,Spanish,Chinese,Vietnamese": 0.22727272727272727, "English,Spanish,Malay,Vietnamese": 0.30113636363636365, "English,Chinese,Malay,Vietnamese": 0.18181818181818182, "Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Spanish,Chinese,Malay,Vietnamese": 0.1875 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.125, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.22727272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1875, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, "AC3_2": 0.4134328514287142, "AC3_3": 0.3202825635221127, "AC3_4": 0.26117508549211566, "AC3_5": 0.22202720705423046, "AC3_6": 0.19427105149051246, "AC3_7": 0.1740650653335523 }, "prompt_2": { "overall_acc": 0.37743506493506496, "language_acc": { "Indonesian": 0.32954545454545453, "English": 0.4147727272727273, "Filipino": 0.2897727272727273, "Spanish": 0.38636363636363635, "Chinese": 0.38636363636363635, "Malay": 0.39204545454545453, "Vietnamese": 0.4431818181818182 }, "consistency_score_2": 0.4599567099567099, "consistency_score_3": 0.2712662337662338, "consistency_score_4": 0.186038961038961, "consistency_score_5": 0.14096320346320346, "consistency_score_6": 0.11607142857142858, "consistency_score_7": 0.10227272727272728, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4431818181818182, "Indonesian,Filipino": 0.375, "Indonesian,Spanish": 0.5113636363636364, "Indonesian,Chinese": 0.3693181818181818, "Indonesian,Malay": 0.5397727272727273, "Indonesian,Vietnamese": 0.4715909090909091, "English,Filipino": 0.42045454545454547, "English,Spanish": 0.5625, "English,Chinese": 0.42045454545454547, "English,Malay": 0.4318181818181818, "English,Vietnamese": 0.4943181818181818, "Filipino,Spanish": 0.4715909090909091, "Filipino,Chinese": 0.32954545454545453, "Filipino,Malay": 0.39204545454545453, "Filipino,Vietnamese": 0.4602272727272727, "Spanish,Chinese": 0.4034090909090909, "Spanish,Malay": 0.5, "Spanish,Vietnamese": 0.5681818181818182, "Chinese,Malay": 0.4431818181818182, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.5625 }, "3_combine": { "Indonesian,English,Filipino": 0.2159090909090909, "Indonesian,English,Spanish": 0.3465909090909091, "Indonesian,English,Chinese": 0.2215909090909091, "Indonesian,English,Malay": 0.2784090909090909, "Indonesian,English,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Spanish": 0.2556818181818182, "Indonesian,Filipino,Chinese": 0.16477272727272727, "Indonesian,Filipino,Malay": 0.23863636363636365, "Indonesian,Filipino,Vietnamese": 0.25, "Indonesian,Spanish,Chinese": 0.23295454545454544, "Indonesian,Spanish,Malay": 0.3522727272727273, "Indonesian,Spanish,Vietnamese": 0.3352272727272727, "Indonesian,Chinese,Malay": 0.26136363636363635, "Indonesian,Chinese,Vietnamese": 0.2556818181818182, "Indonesian,Malay,Vietnamese": 0.36363636363636365, "English,Filipino,Spanish": 0.2840909090909091, "English,Filipino,Chinese": 0.1875, "English,Filipino,Malay": 0.19886363636363635, "English,Filipino,Vietnamese": 0.26704545454545453, "English,Spanish,Chinese": 0.2727272727272727, "English,Spanish,Malay": 0.32386363636363635, "English,Spanish,Vietnamese": 0.3693181818181818, "English,Chinese,Malay": 0.24431818181818182, "English,Chinese,Vietnamese": 0.2840909090909091, "English,Malay,Vietnamese": 0.3125, "Filipino,Spanish,Chinese": 0.21022727272727273, "Filipino,Spanish,Malay": 0.25, "Filipino,Spanish,Vietnamese": 0.3181818181818182, "Filipino,Chinese,Malay": 0.18181818181818182, "Filipino,Chinese,Vietnamese": 0.23295454545454544, "Filipino,Malay,Vietnamese": 0.2784090909090909, "Spanish,Chinese,Malay": 0.25, "Spanish,Chinese,Vietnamese": 0.2727272727272727, "Spanish,Malay,Vietnamese": 0.375, "Chinese,Malay,Vietnamese": 0.3181818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.19318181818181818, "Indonesian,English,Filipino,Chinese": 0.13636363636363635, "Indonesian,English,Filipino,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese": 0.18181818181818182, "Indonesian,English,Spanish,Malay": 0.24431818181818182, "Indonesian,English,Spanish,Vietnamese": 0.24431818181818182, "Indonesian,English,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,English,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, "Indonesian,Filipino,Spanish,Malay": 0.1875, "Indonesian,Filipino,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Chinese,Malay": 0.13068181818181818, "Indonesian,Filipino,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Spanish,Chinese,Malay": 0.18181818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.1875, "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish,Chinese": 0.14772727272727273, "English,Filipino,Spanish,Malay": 0.17613636363636365, "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, "English,Filipino,Chinese,Malay": 0.11931818181818182, "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, "English,Filipino,Malay,Vietnamese": 0.18181818181818182, "English,Spanish,Chinese,Malay": 0.17613636363636365, "English,Spanish,Chinese,Vietnamese": 0.20454545454545456, "English,Spanish,Malay,Vietnamese": 0.26136363636363635, "English,Chinese,Malay,Vietnamese": 0.20454545454545456, "Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.125, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Chinese,Malay": 0.13636363636363635, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Malay,Vietnamese": 0.19886363636363635, "Indonesian,English,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.125, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 } }, "AC3_2": 0.41462979664813365, "AC3_3": 0.31566265936399873, "AC3_4": 0.24923110592844783, "AC3_5": 0.20526478997206252, "AC3_6": 0.17754346800913687, "AC3_7": 0.1609367789235346 }, "prompt_3": { "overall_acc": 0.34983766233766234, "language_acc": { "Indonesian": 0.30113636363636365, "English": 0.4090909090909091, "Filipino": 0.2727272727272727, "Spanish": 0.35795454545454547, "Chinese": 0.3522727272727273, "Malay": 0.3693181818181818, "Vietnamese": 0.38636363636363635 }, "consistency_score_2": 0.4558982683982685, "consistency_score_3": 0.26607142857142857, "consistency_score_4": 0.17938311688311687, "consistency_score_5": 0.1314935064935065, "consistency_score_6": 0.1030844155844156, "consistency_score_7": 0.08522727272727272, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4772727272727273, "Indonesian,Filipino": 0.375, "Indonesian,Spanish": 0.4943181818181818, "Indonesian,Chinese": 0.3806818181818182, "Indonesian,Malay": 0.5170454545454546, "Indonesian,Vietnamese": 0.5056818181818182, "English,Filipino": 0.4375, "English,Spanish": 0.5852272727272727, "English,Chinese": 0.4318181818181818, "English,Malay": 0.44886363636363635, "English,Vietnamese": 0.5056818181818182, "Filipino,Spanish": 0.4602272727272727, "Filipino,Chinese": 0.3522727272727273, "Filipino,Malay": 0.42045454545454547, "Filipino,Vietnamese": 0.4431818181818182, "Spanish,Chinese": 0.38636363636363635, "Spanish,Malay": 0.4943181818181818, "Spanish,Vietnamese": 0.5681818181818182, "Chinese,Malay": 0.39204545454545453, "Chinese,Vietnamese": 0.3806818181818182, "Malay,Vietnamese": 0.5170454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.23863636363636365, "Indonesian,English,Spanish": 0.35795454545454547, "Indonesian,English,Chinese": 0.24431818181818182, "Indonesian,English,Malay": 0.29545454545454547, "Indonesian,English,Vietnamese": 0.30113636363636365, "Indonesian,Filipino,Spanish": 0.24431818181818182, "Indonesian,Filipino,Chinese": 0.16477272727272727, "Indonesian,Filipino,Malay": 0.23863636363636365, "Indonesian,Filipino,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Chinese": 0.21022727272727273, "Indonesian,Spanish,Malay": 0.3522727272727273, "Indonesian,Spanish,Vietnamese": 0.3465909090909091, "Indonesian,Chinese,Malay": 0.2215909090909091, "Indonesian,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,Malay,Vietnamese": 0.3352272727272727, "English,Filipino,Spanish": 0.29545454545454547, "English,Filipino,Chinese": 0.19886363636363635, "English,Filipino,Malay": 0.22727272727272727, "English,Filipino,Vietnamese": 0.2784090909090909, "English,Spanish,Chinese": 0.26136363636363635, "English,Spanish,Malay": 0.3409090909090909, "English,Spanish,Vietnamese": 0.4090909090909091, "English,Chinese,Malay": 0.23863636363636365, "English,Chinese,Vietnamese": 0.23863636363636365, "English,Malay,Vietnamese": 0.30113636363636365, "Filipino,Spanish,Chinese": 0.19318181818181818, "Filipino,Spanish,Malay": 0.26704545454545453, "Filipino,Spanish,Vietnamese": 0.30113636363636365, "Filipino,Chinese,Malay": 0.19886363636363635, "Filipino,Chinese,Vietnamese": 0.19886363636363635, "Filipino,Malay,Vietnamese": 0.2727272727272727, "Spanish,Chinese,Malay": 0.23863636363636365, "Spanish,Chinese,Vietnamese": 0.25, "Spanish,Malay,Vietnamese": 0.36363636363636365, "Chinese,Malay,Vietnamese": 0.23295454545454544 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.20454545454545456, "Indonesian,English,Filipino,Chinese": 0.13068181818181818, "Indonesian,English,Filipino,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Vietnamese": 0.1875, "Indonesian,English,Spanish,Chinese": 0.17045454545454544, "Indonesian,English,Spanish,Malay": 0.2556818181818182, "Indonesian,English,Spanish,Vietnamese": 0.26704545454545453, "Indonesian,English,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,English,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Malay": 0.19318181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Spanish,Chinese,Malay": 0.1534090909090909, "Indonesian,Spanish,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, "Indonesian,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese": 0.14204545454545456, "English,Filipino,Spanish,Malay": 0.19318181818181818, "English,Filipino,Spanish,Vietnamese": 0.24431818181818182, "English,Filipino,Chinese,Malay": 0.13068181818181818, "English,Filipino,Chinese,Vietnamese": 0.14772727272727273, "English,Filipino,Malay,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese,Malay": 0.17613636363636365, "English,Spanish,Chinese,Vietnamese": 0.19886363636363635, "English,Spanish,Malay,Vietnamese": 0.26704545454545453, "English,Chinese,Malay,Vietnamese": 0.1590909090909091, "Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.10227272727272728, "Indonesian,English,Filipino,Spanish,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,English,Filipino,Chinese,Malay": 0.09659090909090909, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.11931818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,English,Spanish,Malay,Vietnamese": 0.20454545454545456, "Indonesian,English,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, "English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "English,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 } }, "AC3_2": 0.395887482179853, "AC3_3": 0.3022582648805129, "AC3_4": 0.23715988761354082, "AC3_5": 0.19114233149047868, "AC3_6": 0.15924510074219936, "AC3_7": 0.1370632631978039 }, "prompt_4": { "overall_acc": 0.3603896103896104, "language_acc": { "Indonesian": 0.32386363636363635, "English": 0.4318181818181818, "Filipino": 0.25, "Spanish": 0.3977272727272727, "Chinese": 0.3522727272727273, "Malay": 0.38636363636363635, "Vietnamese": 0.3806818181818182 }, "consistency_score_2": 0.4667207792207792, "consistency_score_3": 0.27564935064935064, "consistency_score_4": 0.18749999999999994, "consistency_score_5": 0.14015151515151514, "consistency_score_6": 0.11282467532467533, "consistency_score_7": 0.09659090909090909, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4715909090909091, "Indonesian,Filipino": 0.3977272727272727, "Indonesian,Spanish": 0.5340909090909091, "Indonesian,Chinese": 0.3977272727272727, "Indonesian,Malay": 0.5454545454545454, "Indonesian,Vietnamese": 0.5454545454545454, "English,Filipino": 0.4034090909090909, "English,Spanish": 0.5625, "English,Chinese": 0.4431818181818182, "English,Malay": 0.45454545454545453, "English,Vietnamese": 0.5340909090909091, "Filipino,Spanish": 0.4602272727272727, "Filipino,Chinese": 0.3181818181818182, "Filipino,Malay": 0.4034090909090909, "Filipino,Vietnamese": 0.5, "Spanish,Chinese": 0.39204545454545453, "Spanish,Malay": 0.48863636363636365, "Spanish,Vietnamese": 0.5625, "Chinese,Malay": 0.4318181818181818, "Chinese,Vietnamese": 0.3977272727272727, "Malay,Vietnamese": 0.5568181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.24431818181818182, "Indonesian,English,Spanish": 0.35795454545454547, "Indonesian,English,Chinese": 0.23863636363636365, "Indonesian,English,Malay": 0.3068181818181818, "Indonesian,English,Vietnamese": 0.3522727272727273, "Indonesian,Filipino,Spanish": 0.2556818181818182, "Indonesian,Filipino,Chinese": 0.16477272727272727, "Indonesian,Filipino,Malay": 0.25, "Indonesian,Filipino,Vietnamese": 0.2784090909090909, "Indonesian,Spanish,Chinese": 0.24431818181818182, "Indonesian,Spanish,Malay": 0.3465909090909091, "Indonesian,Spanish,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay": 0.26136363636363635, "Indonesian,Chinese,Vietnamese": 0.25, "Indonesian,Malay,Vietnamese": 0.3806818181818182, "English,Filipino,Spanish": 0.2897727272727273, "English,Filipino,Chinese": 0.18181818181818182, "English,Filipino,Malay": 0.22727272727272727, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.2784090909090909, "English,Spanish,Malay": 0.32386363636363635, "English,Spanish,Vietnamese": 0.3977272727272727, "English,Chinese,Malay": 0.24431818181818182, "English,Chinese,Vietnamese": 0.2556818181818182, "English,Malay,Vietnamese": 0.32954545454545453, "Filipino,Spanish,Chinese": 0.17045454545454544, "Filipino,Spanish,Malay": 0.25, "Filipino,Spanish,Vietnamese": 0.32386363636363635, "Filipino,Chinese,Malay": 0.19318181818181818, "Filipino,Chinese,Vietnamese": 0.19886363636363635, "Filipino,Malay,Vietnamese": 0.2897727272727273, "Spanish,Chinese,Malay": 0.23295454545454544, "Spanish,Chinese,Vietnamese": 0.24431818181818182, "Spanish,Malay,Vietnamese": 0.3522727272727273, "Chinese,Malay,Vietnamese": 0.26136363636363635 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.20454545454545456, "Indonesian,English,Filipino,Chinese": 0.13636363636363635, "Indonesian,English,Filipino,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Chinese": 0.1875, "Indonesian,English,Spanish,Malay": 0.2556818181818182, "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Chinese,Vietnamese": 0.1875, "Indonesian,English,Malay,Vietnamese": 0.25, "Indonesian,Filipino,Spanish,Chinese": 0.125, "Indonesian,Filipino,Spanish,Malay": 0.1875, "Indonesian,Filipino,Spanish,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Chinese,Malay": 0.125, "Indonesian,Filipino,Chinese,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Spanish,Chinese,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.18181818181818182, "English,Filipino,Spanish,Chinese": 0.13068181818181818, "English,Filipino,Spanish,Malay": 0.1875, "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, "English,Filipino,Chinese,Malay": 0.13636363636363635, "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay": 0.18181818181818182, "English,Spanish,Chinese,Vietnamese": 0.20454545454545456, "English,Spanish,Malay,Vietnamese": 0.2727272727272727, "English,Chinese,Malay,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, "Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2215909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, "English,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 } }, "AC3_2": 0.40672036496363495, "AC3_3": 0.31237445551064175, "AC3_4": 0.2466666666216454, "AC3_5": 0.2018181817778618, "AC3_6": 0.17184959119578014, "AC3_7": 0.15234942673882385 }, "prompt_5": { "overall_acc": 0.3555194805194805, "language_acc": { "Indonesian": 0.3125, "English": 0.4034090909090909, "Filipino": 0.26136363636363635, "Spanish": 0.3977272727272727, "Chinese": 0.36363636363636365, "Malay": 0.3693181818181818, "Vietnamese": 0.3806818181818182 }, "consistency_score_2": 0.4848484848484847, "consistency_score_3": 0.30600649350649345, "consistency_score_4": 0.22532467532467532, "consistency_score_5": 0.1812770562770563, "consistency_score_6": 0.15422077922077923, "consistency_score_7": 0.13636363636363635, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.4602272727272727, "Indonesian,Filipino": 0.4090909090909091, "Indonesian,Spanish": 0.5170454545454546, "Indonesian,Chinese": 0.42045454545454547, "Indonesian,Malay": 0.5909090909090909, "Indonesian,Vietnamese": 0.5113636363636364, "English,Filipino": 0.42045454545454547, "English,Spanish": 0.6022727272727273, "English,Chinese": 0.4943181818181818, "English,Malay": 0.48295454545454547, "English,Vietnamese": 0.5170454545454546, "Filipino,Spanish": 0.45454545454545453, "Filipino,Chinese": 0.3806818181818182, "Filipino,Malay": 0.4090909090909091, "Filipino,Vietnamese": 0.45454545454545453, "Spanish,Chinese": 0.5170454545454546, "Spanish,Malay": 0.48295454545454547, "Spanish,Vietnamese": 0.5340909090909091, "Chinese,Malay": 0.48295454545454547, "Chinese,Vietnamese": 0.4772727272727273, "Malay,Vietnamese": 0.5625 }, "3_combine": { "Indonesian,English,Filipino": 0.25, "Indonesian,English,Spanish": 0.3693181818181818, "Indonesian,English,Chinese": 0.2727272727272727, "Indonesian,English,Malay": 0.32386363636363635, "Indonesian,English,Vietnamese": 0.32954545454545453, "Indonesian,Filipino,Spanish": 0.2840909090909091, "Indonesian,Filipino,Chinese": 0.2159090909090909, "Indonesian,Filipino,Malay": 0.26704545454545453, "Indonesian,Filipino,Vietnamese": 0.2556818181818182, "Indonesian,Spanish,Chinese": 0.29545454545454547, "Indonesian,Spanish,Malay": 0.3806818181818182, "Indonesian,Spanish,Vietnamese": 0.3693181818181818, "Indonesian,Chinese,Malay": 0.32386363636363635, "Indonesian,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,Malay,Vietnamese": 0.4034090909090909, "English,Filipino,Spanish": 0.30113636363636365, "English,Filipino,Chinese": 0.23295454545454544, "English,Filipino,Malay": 0.25, "English,Filipino,Vietnamese": 0.2840909090909091, "English,Spanish,Chinese": 0.375, "English,Spanish,Malay": 0.3693181818181818, "English,Spanish,Vietnamese": 0.39204545454545453, "English,Chinese,Malay": 0.3181818181818182, "English,Chinese,Vietnamese": 0.32954545454545453, "English,Malay,Vietnamese": 0.3465909090909091, "Filipino,Spanish,Chinese": 0.25, "Filipino,Spanish,Malay": 0.26136363636363635, "Filipino,Spanish,Vietnamese": 0.3068181818181818, "Filipino,Chinese,Malay": 0.2215909090909091, "Filipino,Chinese,Vietnamese": 0.23863636363636365, "Filipino,Malay,Vietnamese": 0.2727272727272727, "Spanish,Chinese,Malay": 0.3125, "Spanish,Chinese,Vietnamese": 0.32954545454545453, "Spanish,Malay,Vietnamese": 0.36363636363636365, "Chinese,Malay,Vietnamese": 0.32386363636363635 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.22727272727272727, "Indonesian,English,Filipino,Chinese": 0.17045454545454544, "Indonesian,English,Filipino,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Vietnamese": 0.20454545454545456, "Indonesian,English,Spanish,Chinese": 0.23863636363636365, "Indonesian,English,Spanish,Malay": 0.29545454545454547, "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, "Indonesian,English,Chinese,Malay": 0.23295454545454544, "Indonesian,English,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,English,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese": 0.1875, "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, "Indonesian,Filipino,Spanish,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Chinese,Malay": 0.17613636363636365, "Indonesian,Filipino,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.2159090909090909, "Indonesian,Spanish,Chinese,Malay": 0.24431818181818182, "Indonesian,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,Spanish,Malay,Vietnamese": 0.3068181818181818, "Indonesian,Chinese,Malay,Vietnamese": 0.25, "English,Filipino,Spanish,Chinese": 0.19886363636363635, "English,Filipino,Spanish,Malay": 0.21022727272727273, "English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "English,Filipino,Chinese,Malay": 0.16477272727272727, "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, "English,Filipino,Malay,Vietnamese": 0.20454545454545456, "English,Spanish,Chinese,Malay": 0.26704545454545453, "English,Spanish,Chinese,Vietnamese": 0.2784090909090909, "English,Spanish,Malay,Vietnamese": 0.29545454545454547, "English,Chinese,Malay,Vietnamese": 0.24431818181818182, "Filipino,Spanish,Chinese,Malay": 0.1875, "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, "Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "Spanish,Chinese,Malay,Vietnamese": 0.25 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.1875, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.21022727272727273, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Indonesian,English,Spanish,Malay,Vietnamese": 0.24431818181818182, "Indonesian,English,Chinese,Malay,Vietnamese": 0.19886363636363635, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 } }, "AC3_2": 0.4102323947296188, "AC3_3": 0.3289100469584865, "AC3_4": 0.2758306533684086, "AC3_5": 0.2401190039351592, "AC3_6": 0.21512325250143222, "AC3_7": 0.1971197119311226 } }, "sg_eval": { "prompt_1": { "accuracy": 0.4854368932038835 }, "prompt_2": { "accuracy": 0.5145631067961165 }, "prompt_3": { "accuracy": 0.5242718446601942 }, "prompt_4": { "accuracy": 0.5242718446601942 }, "prompt_5": { "accuracy": 0.5728155339805825 } }, "cn_eval": { "prompt_1": { "accuracy": 0.3523809523809524 }, "prompt_2": { "accuracy": 0.3619047619047619 }, "prompt_3": { "accuracy": 0.38095238095238093 }, "prompt_4": { "accuracy": 0.3333333333333333 }, "prompt_5": { "accuracy": 0.38095238095238093 } }, "us_eval": { "prompt_1": { "accuracy": 0.4672897196261682 }, "prompt_2": { "accuracy": 0.4485981308411215 }, "prompt_3": { "accuracy": 0.5233644859813084 }, "prompt_4": { "accuracy": 0.4485981308411215 }, "prompt_5": { "accuracy": 0.4672897196261682 } }, "ph_eval": { "prompt_1": { "accuracy": 0.45, "category_acc": { "brand": 0.5, "demographics": 0.2, "biology": 0.6, "history": 0.4666666666666667, "literature": 0.4, "politics": 0.4, "culture": 0.5, "film": 0.4, "law": 0.3, "geography": 0.6 } }, "prompt_2": { "accuracy": 0.4, "category_acc": { "brand": 0.2, "demographics": 0.4, "biology": 0.6, "history": 0.3333333333333333, "literature": 0.4, "politics": 0.2, "culture": 0.7, "film": 0.5, "law": 0.3, "geography": 0.4 } }, "prompt_3": { "accuracy": 0.4, "category_acc": { "brand": 0.1, "demographics": 0.0, "biology": 0.6, "history": 0.4666666666666667, "literature": 0.4, "politics": 0.4, "culture": 0.6, "film": 0.5, "law": 0.2, "geography": 0.5 } }, "prompt_4": { "accuracy": 0.35, "category_acc": { "brand": 0.2, "demographics": 0.0, "biology": 0.6, "history": 0.3333333333333333, "literature": 0.4, "politics": 0.3, "culture": 0.6, "film": 0.3, "law": 0.3, "geography": 0.3 } }, "prompt_5": { "accuracy": 0.35, "category_acc": { "brand": 0.2, "demographics": 0.0, "biology": 0.7, "history": 0.5333333333333333, "literature": 0.4, "politics": 0.1, "culture": 0.5, "film": 0.3, "law": 0.2, "geography": 0.3 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.1516122499780299 }, "prompt_2": { "bleu_score": 0.14823755033441352 }, "prompt_3": { "bleu_score": 0.14739183235034525 }, "prompt_4": { "bleu_score": 0.14953624277303112 }, "prompt_5": { "bleu_score": 0.12282893196393144 } }, "indommlu": { "prompt_1": { "accuracy": 0.39114760664930903, "category_acc": { "History": 0.36947791164658633, "Geography": 0.3816326530612245, "Lampungic": 0.2925170068027211, "Social science": 0.6060100166944908, "Balinese": 0.3184713375796178, "Makassarese": 0.2956989247311828, "Banjarese": 0.3958333333333333, "Chemistry": 0.2291970802919708, "Biology": 0.3479289940828402, "Science": 0.5108359133126935, "Christian religion": 0.4626865671641791, "Art": 0.49750415973377704, "Islam religion": 0.4039829302987198, "Hindu religion": 0.38666666666666666, "Madurese": 0.2711864406779661, "Sport": 0.4527027027027027, "Indonesian language": 0.43088418430884184, "Physics": 0.34545454545454546, "Minangkabau culture": 0.31155778894472363, "Dayak language": 0.30275229357798167, "Sociology": 0.41330645161290325, "Economy": 0.33401639344262296, "Sundanese": 0.32238547968885045, "Javanese": 0.3064516129032258, "Civic education": 0.4263233190271817 } }, "prompt_2": { "accuracy": 0.3895453635089125, "category_acc": { "History": 0.2931726907630522, "Geography": 0.3877551020408163, "Lampungic": 0.2925170068027211, "Social science": 0.5776293823038398, "Balinese": 0.34182590233545646, "Makassarese": 0.34946236559139787, "Banjarese": 0.22916666666666666, "Chemistry": 0.24525547445255474, "Biology": 0.37041420118343193, "Science": 0.48194014447884415, "Christian religion": 0.36318407960199006, "Art": 0.49916805324459235, "Islam religion": 0.41109530583214793, "Hindu religion": 0.43333333333333335, "Madurese": 0.2711864406779661, "Sport": 0.40540540540540543, "Indonesian language": 0.43711083437110837, "Physics": 0.33535353535353535, "Minangkabau culture": 0.36683417085427134, "Dayak language": 0.26605504587155965, "Sociology": 0.40524193548387094, "Economy": 0.3483606557377049, "Sundanese": 0.3388072601555748, "Javanese": 0.29435483870967744, "Civic education": 0.44206008583690987 } }, "prompt_3": { "accuracy": 0.40490019360437945, "category_acc": { "History": 0.3755020080321285, "Geography": 0.39387755102040817, "Lampungic": 0.35374149659863946, "Social science": 0.6076794657762938, "Balinese": 0.3609341825902335, "Makassarese": 0.3387096774193548, "Banjarese": 0.3611111111111111, "Chemistry": 0.24233576642335766, "Biology": 0.37988165680473374, "Science": 0.4953560371517028, "Christian religion": 0.43283582089552236, "Art": 0.49916805324459235, "Islam religion": 0.422475106685633, "Hindu religion": 0.44666666666666666, "Madurese": 0.28135593220338984, "Sport": 0.42567567567567566, "Indonesian language": 0.450186799501868, "Physics": 0.3515151515151515, "Minangkabau culture": 0.34673366834170855, "Dayak language": 0.28440366972477066, "Sociology": 0.40725806451612906, "Economy": 0.35655737704918034, "Sundanese": 0.35695764909248057, "Javanese": 0.29536290322580644, "Civic education": 0.45493562231759654 } }, "prompt_4": { "accuracy": 0.40189598771613594, "category_acc": { "History": 0.3755020080321285, "Geography": 0.4, "Lampungic": 0.3401360544217687, "Social science": 0.5943238731218697, "Balinese": 0.35456475583864117, "Makassarese": 0.3225806451612903, "Banjarese": 0.375, "Chemistry": 0.2291970802919708, "Biology": 0.37514792899408284, "Science": 0.5036119711042312, "Christian religion": 0.43283582089552236, "Art": 0.4925124792013311, "Islam religion": 0.42816500711237554, "Hindu religion": 0.42, "Madurese": 0.2983050847457627, "Sport": 0.4189189189189189, "Indonesian language": 0.44084682440846823, "Physics": 0.3575757575757576, "Minangkabau culture": 0.35175879396984927, "Dayak language": 0.3119266055045872, "Sociology": 0.41935483870967744, "Economy": 0.3729508196721312, "Sundanese": 0.3388072601555748, "Javanese": 0.3094758064516129, "Civic education": 0.4363376251788269 } }, "prompt_5": { "accuracy": 0.3915481674344082, "category_acc": { "History": 0.3815261044176707, "Geography": 0.38571428571428573, "Lampungic": 0.2789115646258503, "Social science": 0.5926544240400667, "Balinese": 0.3673036093418259, "Makassarese": 0.3010752688172043, "Banjarese": 0.3958333333333333, "Chemistry": 0.2116788321167883, "Biology": 0.3585798816568047, "Science": 0.4871001031991744, "Christian religion": 0.4079601990049751, "Art": 0.47753743760399336, "Islam religion": 0.406827880512091, "Hindu religion": 0.43333333333333335, "Madurese": 0.29491525423728815, "Sport": 0.4391891891891892, "Indonesian language": 0.4293275217932752, "Physics": 0.3333333333333333, "Minangkabau culture": 0.35175879396984927, "Dayak language": 0.27522935779816515, "Sociology": 0.39919354838709675, "Economy": 0.3524590163934426, "Sundanese": 0.34312878133102853, "Javanese": 0.29838709677419356, "Civic education": 0.4363376251788269 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.10880435443617271 }, "prompt_2": { "bleu_score": 0.14092288855238244 }, "prompt_3": { "bleu_score": 0.14812578796831388 }, "prompt_4": { "bleu_score": 0.13334769013584924 }, "prompt_5": { "bleu_score": 0.13737276349753164 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.09271405882854561 }, "prompt_2": { "bleu_score": 0.11771122329514913 }, "prompt_3": { "bleu_score": 0.1188780172536138 }, "prompt_4": { "bleu_score": 0.11680891790902595 }, "prompt_5": { "bleu_score": 0.11474357069185724 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.09649623353164899 }, "prompt_2": { "bleu_score": 0.10475100454702978 }, "prompt_3": { "bleu_score": 0.10460986099336168 }, "prompt_4": { "bleu_score": 0.09843583798456172 }, "prompt_5": { "bleu_score": 0.10722583567201201 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.09682685133906475 }, "prompt_2": { "bleu_score": 0.13267327883722826 }, "prompt_3": { "bleu_score": 0.12881593919989232 }, "prompt_4": { "bleu_score": 0.12229829889926759 }, "prompt_5": { "bleu_score": 0.11804875611373819 } }, "mmlu": { "prompt_1": { "accuracy": 0.4515752625437573 }, "prompt_2": { "accuracy": 0.4049008168028005 }, "prompt_3": { "accuracy": 0.46557759626604434 }, "prompt_4": { "accuracy": 0.4620770128354726 }, "prompt_5": { "accuracy": 0.4574095682613769 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.4288165892027172, "category_acc": { "high_school_european_history": 0.2865853658536585, "business_ethics": 0.494949494949495, "clinical_knowledge": 0.4810606060606061, "medical_genetics": 0.45454545454545453, "high_school_us_history": 0.2315270935960591, "high_school_physics": 0.3333333333333333, "high_school_world_history": 0.2669491525423729, "virology": 0.40606060606060607, "high_school_microeconomics": 0.48523206751054854, "econometrics": 0.3274336283185841, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.5598705501618123, "abstract_algebra": 0.1414141414141414, "professional_accounting": 0.3701067615658363, "philosophy": 0.5193548387096775, "professional_medicine": 0.25092250922509224, "nutrition": 0.46229508196721314, "global_facts": 0.29292929292929293, "machine_learning": 0.38738738738738737, "security_studies": 0.4385245901639344, "public_relations": 0.5871559633027523, "professional_psychology": 0.4320785597381342, "prehistory": 0.4984520123839009, "anatomy": 0.4552238805970149, "human_sexuality": 0.5, "college_medicine": 0.36627906976744184, "high_school_government_and_politics": 0.5677083333333334, "college_chemistry": 0.3434343434343434, "logical_fallacies": 0.5246913580246914, "high_school_geography": 0.6446700507614214, "elementary_mathematics": 0.26525198938992045, "human_aging": 0.5720720720720721, "college_mathematics": 0.23232323232323232, "high_school_psychology": 0.6323529411764706, "formal_logic": 0.352, "high_school_statistics": 0.2930232558139535, "international_law": 0.575, "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.36363636363636365, "conceptual_physics": 0.4358974358974359, "miscellaneous": 0.6048593350383632, "high_school_chemistry": 0.3465346534653465, "marketing": 0.759656652360515, "professional_law": 0.3294194390084801, "management": 0.6274509803921569, "college_physics": 0.27722772277227725, "jurisprudence": 0.6355140186915887, "world_religions": 0.611764705882353, "sociology": 0.555, "us_foreign_policy": 0.5353535353535354, "high_school_macroeconomics": 0.4370179948586118, "computer_security": 0.5454545454545454, "moral_scenarios": 0.22371364653243847, "moral_disputes": 0.48985507246376814, "electrical_engineering": 0.4097222222222222, "astronomy": 0.47019867549668876, "college_biology": 0.5314685314685315 } }, "prompt_2": { "accuracy": 0.3894887379335002, "category_acc": { "high_school_european_history": 0.2926829268292683, "business_ethics": 0.48484848484848486, "clinical_knowledge": 0.4166666666666667, "medical_genetics": 0.3939393939393939, "high_school_us_history": 0.24630541871921183, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.2627118644067797, "virology": 0.3696969696969697, "high_school_microeconomics": 0.45147679324894513, "econometrics": 0.2831858407079646, "college_computer_science": 0.3939393939393939, "high_school_biology": 0.5145631067961165, "abstract_algebra": 0.1919191919191919, "professional_accounting": 0.37722419928825623, "philosophy": 0.44193548387096776, "professional_medicine": 0.24723247232472326, "nutrition": 0.3836065573770492, "global_facts": 0.2828282828282828, "machine_learning": 0.34234234234234234, "security_studies": 0.45901639344262296, "public_relations": 0.5504587155963303, "professional_psychology": 0.40589198036006546, "prehistory": 0.4086687306501548, "anatomy": 0.44029850746268656, "human_sexuality": 0.3923076923076923, "college_medicine": 0.3488372093023256, "high_school_government_and_politics": 0.5572916666666666, "college_chemistry": 0.32323232323232326, "logical_fallacies": 0.4876543209876543, "high_school_geography": 0.5279187817258884, "elementary_mathematics": 0.23607427055702918, "human_aging": 0.4594594594594595, "college_mathematics": 0.30303030303030304, "high_school_psychology": 0.5974264705882353, "formal_logic": 0.312, "high_school_statistics": 0.2837209302325581, "international_law": 0.6083333333333333, "high_school_mathematics": 0.20446096654275092, "high_school_computer_science": 0.3333333333333333, "conceptual_physics": 0.28205128205128205, "miscellaneous": 0.4322250639386189, "high_school_chemistry": 0.25742574257425743, "marketing": 0.7253218884120172, "professional_law": 0.33659491193737767, "management": 0.5784313725490197, "college_physics": 0.2871287128712871, "jurisprudence": 0.5327102803738317, "world_religions": 0.4294117647058823, "sociology": 0.53, "us_foreign_policy": 0.41414141414141414, "high_school_macroeconomics": 0.4138817480719794, "computer_security": 0.47474747474747475, "moral_scenarios": 0.27740492170022374, "moral_disputes": 0.4434782608695652, "electrical_engineering": 0.2777777777777778, "astronomy": 0.4304635761589404, "college_biology": 0.46153846153846156 } }, "prompt_3": { "accuracy": 0.4429746156596353, "category_acc": { "high_school_european_history": 0.2865853658536585, "business_ethics": 0.5050505050505051, "clinical_knowledge": 0.4583333333333333, "medical_genetics": 0.494949494949495, "high_school_us_history": 0.3103448275862069, "high_school_physics": 0.26, "high_school_world_history": 0.3432203389830508, "virology": 0.38181818181818183, "high_school_microeconomics": 0.5147679324894515, "econometrics": 0.30973451327433627, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.5728155339805825, "abstract_algebra": 0.20202020202020202, "professional_accounting": 0.3879003558718861, "philosophy": 0.5419354838709678, "professional_medicine": 0.2915129151291513, "nutrition": 0.4360655737704918, "global_facts": 0.32323232323232326, "machine_learning": 0.38738738738738737, "security_studies": 0.4426229508196721, "public_relations": 0.6055045871559633, "professional_psychology": 0.4386252045826514, "prehistory": 0.4953560371517028, "anatomy": 0.4925373134328358, "human_sexuality": 0.5, "college_medicine": 0.37790697674418605, "high_school_government_and_politics": 0.5833333333333334, "college_chemistry": 0.3333333333333333, "logical_fallacies": 0.5432098765432098, "high_school_geography": 0.649746192893401, "elementary_mathematics": 0.259946949602122, "human_aging": 0.6126126126126126, "college_mathematics": 0.31313131313131315, "high_school_psychology": 0.65625, "formal_logic": 0.32, "high_school_statistics": 0.2372093023255814, "international_law": 0.625, "high_school_mathematics": 0.22304832713754646, "high_school_computer_science": 0.41414141414141414, "conceptual_physics": 0.45726495726495725, "miscellaneous": 0.6508951406649617, "high_school_chemistry": 0.38613861386138615, "marketing": 0.7553648068669528, "professional_law": 0.33724722765818654, "management": 0.6372549019607843, "college_physics": 0.2871287128712871, "jurisprudence": 0.6261682242990654, "world_religions": 0.6588235294117647, "sociology": 0.56, "us_foreign_policy": 0.6363636363636364, "high_school_macroeconomics": 0.4498714652956298, "computer_security": 0.5252525252525253, "moral_scenarios": 0.25838926174496646, "moral_disputes": 0.5246376811594203, "electrical_engineering": 0.3888888888888889, "astronomy": 0.4370860927152318, "college_biology": 0.5594405594405595 } }, "prompt_4": { "accuracy": 0.44268859492313195, "category_acc": { "high_school_european_history": 0.2865853658536585, "business_ethics": 0.5050505050505051, "clinical_knowledge": 0.4772727272727273, "medical_genetics": 0.47474747474747475, "high_school_us_history": 0.30049261083743845, "high_school_physics": 0.2866666666666667, "high_school_world_history": 0.2627118644067797, "virology": 0.38181818181818183, "high_school_microeconomics": 0.4978902953586498, "econometrics": 0.3008849557522124, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.56957928802589, "abstract_algebra": 0.2222222222222222, "professional_accounting": 0.36298932384341637, "philosophy": 0.535483870967742, "professional_medicine": 0.2693726937269373, "nutrition": 0.46885245901639344, "global_facts": 0.24242424242424243, "machine_learning": 0.34234234234234234, "security_studies": 0.45491803278688525, "public_relations": 0.6330275229357798, "professional_psychology": 0.4500818330605565, "prehistory": 0.5015479876160991, "anatomy": 0.4925373134328358, "human_sexuality": 0.5076923076923077, "college_medicine": 0.42441860465116277, "high_school_government_and_politics": 0.5989583333333334, "college_chemistry": 0.3838383838383838, "logical_fallacies": 0.5308641975308642, "high_school_geography": 0.649746192893401, "elementary_mathematics": 0.2625994694960212, "human_aging": 0.5990990990990991, "college_mathematics": 0.2222222222222222, "high_school_psychology": 0.6599264705882353, "formal_logic": 0.32, "high_school_statistics": 0.30697674418604654, "international_law": 0.6166666666666667, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.46464646464646464, "conceptual_physics": 0.47863247863247865, "miscellaneous": 0.6572890025575447, "high_school_chemistry": 0.3316831683168317, "marketing": 0.7639484978540773, "professional_law": 0.32811480756686234, "management": 0.6666666666666666, "college_physics": 0.297029702970297, "jurisprudence": 0.616822429906542, "world_religions": 0.6588235294117647, "sociology": 0.53, "us_foreign_policy": 0.6363636363636364, "high_school_macroeconomics": 0.442159383033419, "computer_security": 0.5252525252525253, "moral_scenarios": 0.2505592841163311, "moral_disputes": 0.5246376811594203, "electrical_engineering": 0.4166666666666667, "astronomy": 0.4503311258278146, "college_biology": 0.5804195804195804 } }, "prompt_5": { "accuracy": 0.4372542009295674, "category_acc": { "high_school_european_history": 0.2073170731707317, "business_ethics": 0.5050505050505051, "clinical_knowledge": 0.5, "medical_genetics": 0.48484848484848486, "high_school_us_history": 0.27586206896551724, "high_school_physics": 0.24666666666666667, "high_school_world_history": 0.2754237288135593, "virology": 0.37575757575757573, "high_school_microeconomics": 0.4767932489451477, "econometrics": 0.34513274336283184, "college_computer_science": 0.3434343434343434, "high_school_biology": 0.5922330097087378, "abstract_algebra": 0.26262626262626265, "professional_accounting": 0.3701067615658363, "philosophy": 0.5258064516129032, "professional_medicine": 0.22878228782287824, "nutrition": 0.42295081967213116, "global_facts": 0.25252525252525254, "machine_learning": 0.35135135135135137, "security_studies": 0.4713114754098361, "public_relations": 0.6238532110091743, "professional_psychology": 0.4386252045826514, "prehistory": 0.4984520123839009, "anatomy": 0.5074626865671642, "human_sexuality": 0.5076923076923077, "college_medicine": 0.4186046511627907, "high_school_government_and_politics": 0.640625, "college_chemistry": 0.3838383838383838, "logical_fallacies": 0.5246913580246914, "high_school_geography": 0.6395939086294417, "elementary_mathematics": 0.2413793103448276, "human_aging": 0.5720720720720721, "college_mathematics": 0.18181818181818182, "high_school_psychology": 0.6544117647058824, "formal_logic": 0.32, "high_school_statistics": 0.28837209302325584, "international_law": 0.6083333333333333, "high_school_mathematics": 0.21189591078066913, "high_school_computer_science": 0.41414141414141414, "conceptual_physics": 0.46153846153846156, "miscellaneous": 0.6381074168797954, "high_school_chemistry": 0.3712871287128713, "marketing": 0.7553648068669528, "professional_law": 0.32811480756686234, "management": 0.6372549019607843, "college_physics": 0.31683168316831684, "jurisprudence": 0.616822429906542, "world_religions": 0.6705882352941176, "sociology": 0.565, "us_foreign_policy": 0.6464646464646465, "high_school_macroeconomics": 0.4498714652956298, "computer_security": 0.5555555555555556, "moral_scenarios": 0.25838926174496646, "moral_disputes": 0.518840579710145, "electrical_engineering": 0.4236111111111111, "astronomy": 0.423841059602649, "college_biology": 0.5524475524475524 } } }, "c_eval": { "prompt_1": { "accuracy": 0.3551263001485884 }, "prompt_2": { "accuracy": 0.34026745913818723 }, "prompt_3": { "accuracy": 0.3187221396731055 }, "prompt_4": { "accuracy": 0.3187221396731055 }, "prompt_5": { "accuracy": 0.3476968796433878 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.3287671232876712, "category_acc": { "computer_network": 0.16666666666666666, "operating_system": 0.5416666666666666, "computer_architecture": 0.4230769230769231, "college_programming": 0.4523809523809524, "college_physics": 0.16666666666666666, "college_chemistry": 0.1724137931034483, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.14285714285714285, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.25, "middle_school_mathematics": 0.2916666666666667, "middle_school_biology": 0.4230769230769231, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.3, "business_administration": 0.21052631578947367, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.41379310344827586, "education_science": 0.5588235294117647, "teacher_qualification": 0.4489795918367347, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.25, "ideological_and_moral_cultivation": 0.4166666666666667, "logic": 0.4444444444444444, "law": 0.1724137931034483, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.2894736842105263, "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.2857142857142857, "high_school_chinese": 0.25, "high_school_history": 0.2, "middle_school_history": 0.2962962962962963, "civil_servant": 0.2692307692307692, "sports_science": 0.5, "plant_protection": 0.5555555555555556, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.2962962962962963, "urban_and_rural_planner": 0.37254901960784315, "accountant": 0.25925925925925924, "fire_engineer": 0.3055555555555556, "environmental_impact_assessment_engineer": 0.3333333333333333, "tax_accountant": 0.3333333333333333, "physician": 0.35185185185185186 } }, "prompt_2": { "accuracy": 0.3412204234122042, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.4583333333333333, "computer_architecture": 0.23076923076923078, "college_programming": 0.40476190476190477, "college_physics": 0.125, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.25, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.4166666666666667, "high_school_chemistry": 0.25, "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.32, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.38333333333333336, "business_administration": 0.23684210526315788, "marxism": 0.4166666666666667, "mao_zedong_thought": 0.41379310344827586, "education_science": 0.6176470588235294, "teacher_qualification": 0.4897959183673469, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.32142857142857145, "ideological_and_moral_cultivation": 0.5, "logic": 0.37037037037037035, "law": 0.2413793103448276, "chinese_language_and_literature": 0.25, "art_studies": 0.34210526315789475, "professional_tour_guide": 0.38235294117647056, "legal_professional": 0.17857142857142858, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.32, "middle_school_history": 0.25925925925925924, "civil_servant": 0.38461538461538464, "sports_science": 0.4166666666666667, "plant_protection": 0.5185185185185185, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3137254901960784, "accountant": 0.18518518518518517, "fire_engineer": 0.3611111111111111, "environmental_impact_assessment_engineer": 0.2222222222222222, "tax_accountant": 0.37037037037037035, "physician": 0.37037037037037035 } }, "prompt_3": { "accuracy": 0.3468244084682441, "category_acc": { "computer_network": 0.25, "operating_system": 0.4166666666666667, "computer_architecture": 0.46153846153846156, "college_programming": 0.2857142857142857, "college_physics": 0.20833333333333334, "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.25, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.19047619047619047, "metrology_engineer": 0.41379310344827586, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.375, "middle_school_mathematics": 0.4166666666666667, "middle_school_biology": 0.4230769230769231, "middle_school_physics": 0.3333333333333333, "middle_school_chemistry": 0.56, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.3333333333333333, "business_administration": 0.23684210526315788, "marxism": 0.375, "mao_zedong_thought": 0.4482758620689655, "education_science": 0.7058823529411765, "teacher_qualification": 0.5102040816326531, "high_school_politics": 0.2916666666666667, "high_school_geography": 0.4583333333333333, "middle_school_politics": 0.2692307692307692, "middle_school_geography": 0.23529411764705882, "modern_chinese_history": 0.32142857142857145, "ideological_and_moral_cultivation": 0.5416666666666666, "logic": 0.48148148148148145, "law": 0.41379310344827586, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.2894736842105263, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.25, "high_school_chinese": 0.3333333333333333, "high_school_history": 0.36, "middle_school_history": 0.14814814814814814, "civil_servant": 0.28846153846153844, "sports_science": 0.375, "plant_protection": 0.5555555555555556, "basic_medicine": 0.16666666666666666, "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3333333333333333, "accountant": 0.35185185185185186, "fire_engineer": 0.3333333333333333, "environmental_impact_assessment_engineer": 0.4166666666666667, "tax_accountant": 0.35185185185185186, "physician": 0.4074074074074074 } }, "prompt_4": { "accuracy": 0.338107098381071, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.4583333333333333, "computer_architecture": 0.23076923076923078, "college_programming": 0.40476190476190477, "college_physics": 0.25, "college_chemistry": 0.13793103448275862, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.13043478260869565, "high_school_physics": 0.4583333333333333, "high_school_chemistry": 0.08333333333333333, "high_school_biology": 0.375, "middle_school_mathematics": 0.25, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.625, "middle_school_chemistry": 0.36, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.35, "business_administration": 0.18421052631578946, "marxism": 0.375, "mao_zedong_thought": 0.41379310344827586, "education_science": 0.5588235294117647, "teacher_qualification": 0.46938775510204084, "high_school_politics": 0.25, "high_school_geography": 0.3333333333333333, "middle_school_politics": 0.5, "middle_school_geography": 0.4117647058823529, "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.4583333333333333, "logic": 0.25925925925925924, "law": 0.20689655172413793, "chinese_language_and_literature": 0.2857142857142857, "art_studies": 0.3157894736842105, "professional_tour_guide": 0.38235294117647056, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.4166666666666667, "high_school_history": 0.32, "middle_school_history": 0.2222222222222222, "civil_servant": 0.40384615384615385, "sports_science": 0.20833333333333334, "plant_protection": 0.5555555555555556, "basic_medicine": 0.20833333333333334, "clinical_medicine": 0.37037037037037035, "urban_and_rural_planner": 0.4117647058823529, "accountant": 0.3148148148148148, "fire_engineer": 0.2777777777777778, "environmental_impact_assessment_engineer": 0.3888888888888889, "tax_accountant": 0.25925925925925924, "physician": 0.3148148148148148 } }, "prompt_5": { "accuracy": 0.3443337484433375, "category_acc": { "computer_network": 0.2916666666666667, "operating_system": 0.5416666666666666, "computer_architecture": 0.38461538461538464, "college_programming": 0.3333333333333333, "college_physics": 0.2916666666666667, "college_chemistry": 0.20689655172413793, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.34782608695652173, "discrete_mathematics": 0.19047619047619047, "electrical_engineer": 0.2619047619047619, "metrology_engineer": 0.5172413793103449, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.375, "high_school_chemistry": 0.2916666666666667, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.25, "middle_school_biology": 0.46153846153846156, "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.4, "veterinary_medicine": 0.35714285714285715, "college_economics": 0.4166666666666667, "business_administration": 0.23684210526315788, "marxism": 0.375, "mao_zedong_thought": 0.5172413793103449, "education_science": 0.5882352941176471, "teacher_qualification": 0.4489795918367347, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.4166666666666667, "middle_school_politics": 0.38461538461538464, "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.17857142857142858, "ideological_and_moral_cultivation": 0.7083333333333334, "logic": 0.37037037037037035, "law": 0.20689655172413793, "chinese_language_and_literature": 0.32142857142857145, "art_studies": 0.21052631578947367, "professional_tour_guide": 0.35294117647058826, "legal_professional": 0.21428571428571427, "high_school_chinese": 0.375, "high_school_history": 0.28, "middle_school_history": 0.25925925925925924, "civil_servant": 0.2692307692307692, "sports_science": 0.4583333333333333, "plant_protection": 0.5185185185185185, "basic_medicine": 0.375, "clinical_medicine": 0.4444444444444444, "urban_and_rural_planner": 0.3137254901960784, "accountant": 0.2777777777777778, "fire_engineer": 0.3333333333333333, "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.3333333333333333, "physician": 0.35185185185185186 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.3942652329749104 }, "prompt_2": { "accuracy": 0.4050179211469534 }, "prompt_3": { "accuracy": 0.3835125448028674 }, "prompt_4": { "accuracy": 0.36917562724014336 }, "prompt_5": { "accuracy": 0.3333333333333333 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.35598342255223625, "category_acc": { "agronomy": 0.40236686390532544, "anatomy": 0.27702702702702703, "ancient_chinese": 0.2865853658536585, "arts": 0.36875, "astronomy": 0.30303030303030304, "business_ethics": 0.44976076555023925, "chinese_civil_service_exam": 0.29375, "chinese_driving_rule": 0.4351145038167939, "chinese_food_culture": 0.29411764705882354, "chinese_foreign_policy": 0.2616822429906542, "chinese_history": 0.33746130030959753, "chinese_literature": 0.2647058823529412, "chinese_teacher_qualification": 0.41899441340782123, "clinical_knowledge": 0.2742616033755274, "college_actuarial_science": 0.29245283018867924, "college_education": 0.45794392523364486, "college_engineering_hydrology": 0.3490566037735849, "college_law": 0.28703703703703703, "college_mathematics": 0.23809523809523808, "college_medical_statistics": 0.33962264150943394, "college_medicine": 0.27106227106227104, "computer_science": 0.37745098039215685, "computer_security": 0.4502923976608187, "conceptual_physics": 0.36054421768707484, "construction_project_management": 0.33093525179856115, "economics": 0.39622641509433965, "education": 0.43558282208588955, "electrical_engineering": 0.4127906976744186, "elementary_chinese": 0.2698412698412698, "elementary_commonsense": 0.35858585858585856, "elementary_information_and_technology": 0.592436974789916, "elementary_mathematics": 0.24782608695652175, "ethnology": 0.362962962962963, "food_science": 0.3706293706293706, "genetics": 0.2840909090909091, "global_facts": 0.37583892617449666, "high_school_biology": 0.31952662721893493, "high_school_chemistry": 0.29545454545454547, "high_school_geography": 0.3135593220338983, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.24545454545454545, "high_school_politics": 0.3146853146853147, "human_sexuality": 0.3412698412698413, "international_law": 0.32972972972972975, "journalism": 0.4127906976744186, "jurisprudence": 0.34793187347931875, "legal_and_moral_basis": 0.616822429906542, "logical": 0.3983739837398374, "machine_learning": 0.29508196721311475, "management": 0.40476190476190477, "marketing": 0.5, "marxist_theory": 0.4126984126984127, "modern_chinese": 0.3017241379310345, "nutrition": 0.33793103448275863, "philosophy": 0.4666666666666667, "professional_accounting": 0.42857142857142855, "professional_law": 0.2938388625592417, "professional_medicine": 0.28191489361702127, "professional_psychology": 0.4267241379310345, "public_relations": 0.3735632183908046, "security_study": 0.362962962962963, "sociology": 0.3893805309734513, "sports_science": 0.40606060606060607, "traditional_chinese_medicine": 0.2810810810810811, "virology": 0.4260355029585799, "world_history": 0.2795031055900621, "world_religions": 0.39375 } }, "prompt_2": { "accuracy": 0.3518390606112934, "category_acc": { "agronomy": 0.3727810650887574, "anatomy": 0.25675675675675674, "ancient_chinese": 0.3170731707317073, "arts": 0.3375, "astronomy": 0.30303030303030304, "business_ethics": 0.45454545454545453, "chinese_civil_service_exam": 0.3125, "chinese_driving_rule": 0.48091603053435117, "chinese_food_culture": 0.29411764705882354, "chinese_foreign_policy": 0.2897196261682243, "chinese_history": 0.34365325077399383, "chinese_literature": 0.27450980392156865, "chinese_teacher_qualification": 0.4134078212290503, "clinical_knowledge": 0.27848101265822783, "college_actuarial_science": 0.2641509433962264, "college_education": 0.4766355140186916, "college_engineering_hydrology": 0.330188679245283, "college_law": 0.25925925925925924, "college_mathematics": 0.2571428571428571, "college_medical_statistics": 0.32075471698113206, "college_medicine": 0.2893772893772894, "computer_science": 0.37745098039215685, "computer_security": 0.40350877192982454, "conceptual_physics": 0.3673469387755102, "construction_project_management": 0.31654676258992803, "economics": 0.42138364779874216, "education": 0.4171779141104294, "electrical_engineering": 0.4011627906976744, "elementary_chinese": 0.28174603174603174, "elementary_commonsense": 0.35353535353535354, "elementary_information_and_technology": 0.6008403361344538, "elementary_mathematics": 0.28695652173913044, "ethnology": 0.2814814814814815, "food_science": 0.40559440559440557, "genetics": 0.29545454545454547, "global_facts": 0.35570469798657717, "high_school_biology": 0.33136094674556216, "high_school_chemistry": 0.16666666666666666, "high_school_geography": 0.3135593220338983, "high_school_mathematics": 0.22560975609756098, "high_school_physics": 0.3090909090909091, "high_school_politics": 0.25874125874125875, "human_sexuality": 0.3333333333333333, "international_law": 0.33513513513513515, "journalism": 0.4127906976744186, "jurisprudence": 0.34793187347931875, "legal_and_moral_basis": 0.6121495327102804, "logical": 0.34959349593495936, "machine_learning": 0.26229508196721313, "management": 0.41904761904761906, "marketing": 0.45, "marxist_theory": 0.41798941798941797, "modern_chinese": 0.2672413793103448, "nutrition": 0.2896551724137931, "philosophy": 0.42857142857142855, "professional_accounting": 0.4114285714285714, "professional_law": 0.2796208530805687, "professional_medicine": 0.27925531914893614, "professional_psychology": 0.4525862068965517, "public_relations": 0.40229885057471265, "security_study": 0.3851851851851852, "sociology": 0.37610619469026546, "sports_science": 0.4, "traditional_chinese_medicine": 0.2756756756756757, "virology": 0.4319526627218935, "world_history": 0.2670807453416149, "world_religions": 0.35625 } }, "prompt_3": { "accuracy": 0.35252978760145054, "category_acc": { "agronomy": 0.35502958579881655, "anatomy": 0.24324324324324326, "ancient_chinese": 0.2865853658536585, "arts": 0.36875, "astronomy": 0.24242424242424243, "business_ethics": 0.430622009569378, "chinese_civil_service_exam": 0.2875, "chinese_driving_rule": 0.42748091603053434, "chinese_food_culture": 0.3088235294117647, "chinese_foreign_policy": 0.27102803738317754, "chinese_history": 0.3653250773993808, "chinese_literature": 0.2696078431372549, "chinese_teacher_qualification": 0.4134078212290503, "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.2358490566037736, "college_education": 0.4766355140186916, "college_engineering_hydrology": 0.39622641509433965, "college_law": 0.23148148148148148, "college_mathematics": 0.2761904761904762, "college_medical_statistics": 0.330188679245283, "college_medicine": 0.2893772893772894, "computer_science": 0.37254901960784315, "computer_security": 0.4327485380116959, "conceptual_physics": 0.3401360544217687, "construction_project_management": 0.34532374100719426, "economics": 0.3836477987421384, "education": 0.39263803680981596, "electrical_engineering": 0.43023255813953487, "elementary_chinese": 0.27380952380952384, "elementary_commonsense": 0.30808080808080807, "elementary_information_and_technology": 0.5630252100840336, "elementary_mathematics": 0.2217391304347826, "ethnology": 0.32592592592592595, "food_science": 0.40559440559440557, "genetics": 0.29545454545454547, "global_facts": 0.3691275167785235, "high_school_biology": 0.2781065088757396, "high_school_chemistry": 0.32575757575757575, "high_school_geography": 0.3389830508474576, "high_school_mathematics": 0.25609756097560976, "high_school_physics": 0.36363636363636365, "high_school_politics": 0.2867132867132867, "human_sexuality": 0.3492063492063492, "international_law": 0.32972972972972975, "journalism": 0.38953488372093026, "jurisprudence": 0.3309002433090024, "legal_and_moral_basis": 0.6214953271028038, "logical": 0.42276422764227645, "machine_learning": 0.3360655737704918, "management": 0.4238095238095238, "marketing": 0.49444444444444446, "marxist_theory": 0.43386243386243384, "modern_chinese": 0.2672413793103448, "nutrition": 0.32413793103448274, "philosophy": 0.47619047619047616, "professional_accounting": 0.41714285714285715, "professional_law": 0.27014218009478674, "professional_medicine": 0.26595744680851063, "professional_psychology": 0.40948275862068967, "public_relations": 0.39655172413793105, "security_study": 0.37777777777777777, "sociology": 0.3938053097345133, "sports_science": 0.37575757575757573, "traditional_chinese_medicine": 0.31891891891891894, "virology": 0.42011834319526625, "world_history": 0.32298136645962733, "world_religions": 0.375 } }, "prompt_4": { "accuracy": 0.34907615265066483, "category_acc": { "agronomy": 0.3727810650887574, "anatomy": 0.22972972972972974, "ancient_chinese": 0.2926829268292683, "arts": 0.36875, "astronomy": 0.2606060606060606, "business_ethics": 0.42105263157894735, "chinese_civil_service_exam": 0.28125, "chinese_driving_rule": 0.3893129770992366, "chinese_food_culture": 0.3014705882352941, "chinese_foreign_policy": 0.29906542056074764, "chinese_history": 0.34365325077399383, "chinese_literature": 0.30392156862745096, "chinese_teacher_qualification": 0.39664804469273746, "clinical_knowledge": 0.3206751054852321, "college_actuarial_science": 0.22641509433962265, "college_education": 0.3925233644859813, "college_engineering_hydrology": 0.32075471698113206, "college_law": 0.32407407407407407, "college_mathematics": 0.26666666666666666, "college_medical_statistics": 0.3490566037735849, "college_medicine": 0.31135531135531136, "computer_science": 0.3431372549019608, "computer_security": 0.4502923976608187, "conceptual_physics": 0.3469387755102041, "construction_project_management": 0.37410071942446044, "economics": 0.41509433962264153, "education": 0.44171779141104295, "electrical_engineering": 0.38953488372093026, "elementary_chinese": 0.2857142857142857, "elementary_commonsense": 0.30303030303030304, "elementary_information_and_technology": 0.5672268907563025, "elementary_mathematics": 0.23043478260869565, "ethnology": 0.3111111111111111, "food_science": 0.4125874125874126, "genetics": 0.26704545454545453, "global_facts": 0.348993288590604, "high_school_biology": 0.2958579881656805, "high_school_chemistry": 0.2727272727272727, "high_school_geography": 0.2966101694915254, "high_school_mathematics": 0.2621951219512195, "high_school_physics": 0.2909090909090909, "high_school_politics": 0.2517482517482518, "human_sexuality": 0.3412698412698413, "international_law": 0.34594594594594597, "journalism": 0.3488372093023256, "jurisprudence": 0.36253041362530414, "legal_and_moral_basis": 0.602803738317757, "logical": 0.3983739837398374, "machine_learning": 0.30327868852459017, "management": 0.41904761904761906, "marketing": 0.4722222222222222, "marxist_theory": 0.3968253968253968, "modern_chinese": 0.28448275862068967, "nutrition": 0.32413793103448274, "philosophy": 0.49523809523809526, "professional_accounting": 0.4, "professional_law": 0.2559241706161137, "professional_medicine": 0.26861702127659576, "professional_psychology": 0.4224137931034483, "public_relations": 0.39655172413793105, "security_study": 0.362962962962963, "sociology": 0.3938053097345133, "sports_science": 0.4, "traditional_chinese_medicine": 0.2810810810810811, "virology": 0.378698224852071, "world_history": 0.2857142857142857, "world_religions": 0.3625 } }, "prompt_5": { "accuracy": 0.3443274045933345, "category_acc": { "agronomy": 0.39644970414201186, "anatomy": 0.21621621621621623, "ancient_chinese": 0.2926829268292683, "arts": 0.38125, "astronomy": 0.21818181818181817, "business_ethics": 0.4449760765550239, "chinese_civil_service_exam": 0.2625, "chinese_driving_rule": 0.4961832061068702, "chinese_food_culture": 0.2647058823529412, "chinese_foreign_policy": 0.29906542056074764, "chinese_history": 0.32507739938080493, "chinese_literature": 0.3235294117647059, "chinese_teacher_qualification": 0.41899441340782123, "clinical_knowledge": 0.25316455696202533, "college_actuarial_science": 0.19811320754716982, "college_education": 0.4766355140186916, "college_engineering_hydrology": 0.3490566037735849, "college_law": 0.2777777777777778, "college_mathematics": 0.21904761904761905, "college_medical_statistics": 0.3018867924528302, "college_medicine": 0.2783882783882784, "computer_science": 0.36764705882352944, "computer_security": 0.43859649122807015, "conceptual_physics": 0.3333333333333333, "construction_project_management": 0.33093525179856115, "economics": 0.42138364779874216, "education": 0.4294478527607362, "electrical_engineering": 0.3953488372093023, "elementary_chinese": 0.25, "elementary_commonsense": 0.3181818181818182, "elementary_information_and_technology": 0.5504201680672269, "elementary_mathematics": 0.24782608695652175, "ethnology": 0.37037037037037035, "food_science": 0.3706293706293706, "genetics": 0.32954545454545453, "global_facts": 0.33557046979865773, "high_school_biology": 0.33136094674556216, "high_school_chemistry": 0.25757575757575757, "high_school_geography": 0.3050847457627119, "high_school_mathematics": 0.24390243902439024, "high_school_physics": 0.32727272727272727, "high_school_politics": 0.26573426573426573, "human_sexuality": 0.373015873015873, "international_law": 0.3027027027027027, "journalism": 0.38953488372093026, "jurisprudence": 0.32116788321167883, "legal_and_moral_basis": 0.5841121495327103, "logical": 0.34959349593495936, "machine_learning": 0.36065573770491804, "management": 0.4, "marketing": 0.4722222222222222, "marxist_theory": 0.43386243386243384, "modern_chinese": 0.28448275862068967, "nutrition": 0.30344827586206896, "philosophy": 0.4666666666666667, "professional_accounting": 0.38285714285714284, "professional_law": 0.3033175355450237, "professional_medicine": 0.24468085106382978, "professional_psychology": 0.40086206896551724, "public_relations": 0.3505747126436782, "security_study": 0.3333333333333333, "sociology": 0.38495575221238937, "sports_science": 0.38181818181818183, "traditional_chinese_medicine": 0.2864864864864865, "virology": 0.3905325443786982, "world_history": 0.2857142857142857, "world_religions": 0.35625 } } }, "zbench": { "prompt_1": { "accuracy": 0.18181818181818182 }, "prompt_2": { "accuracy": 0.2727272727272727 }, "prompt_3": { "accuracy": 0.15151515151515152 }, "prompt_4": { "accuracy": 0.21212121212121213 }, "prompt_5": { "accuracy": 0.2727272727272727 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.44545454545454544 }, "prompt_2": { "accuracy": 0.42954545454545456 }, "prompt_3": { "accuracy": 0.4090909090909091 }, "prompt_4": { "accuracy": 0.4318181818181818 }, "prompt_5": { "accuracy": 0.4113636363636364 } }, "ocnli": { "prompt_1": { "accuracy": 0.3813559322033898 }, "prompt_2": { "accuracy": 0.36542372881355933 }, "prompt_3": { "accuracy": 0.3742372881355932 }, "prompt_4": { "accuracy": 0.3505084745762712 }, "prompt_5": { "accuracy": 0.3701694915254237 } }, "c3": { "prompt_1": { "accuracy": 0.5822737471952132 }, "prompt_2": { "accuracy": 0.6136873597606581 }, "prompt_3": { "accuracy": 0.5961106955871354 }, "prompt_4": { "accuracy": 0.5867614061331339 }, "prompt_5": { "accuracy": 0.5905011219147345 } }, "dream": { "prompt_1": { "accuracy": 0.6913277804997551 }, "prompt_2": { "accuracy": 0.6829985301322881 }, "prompt_3": { "accuracy": 0.6766291033806957 }, "prompt_4": { "accuracy": 0.6874081332680059 }, "prompt_5": { "accuracy": 0.6761391474767271 } }, "samsum": { "prompt_1": { "rouge1": 0.15489598387752065, "rouge2": 0.05783232912129048, "rougeL": 0.11671666590863215, "avg_rouge": 0.10981499296914776 }, "prompt_2": { "rouge1": 0.1557839271596062, "rouge2": 0.05529561549724459, "rougeL": 0.11679520097223604, "avg_rouge": 0.10929158120969562 }, "prompt_3": { "rouge1": 0.15749480681429107, "rouge2": 0.0562182055296114, "rougeL": 0.1173178398082738, "avg_rouge": 0.11034361738405875 }, "prompt_4": { "rouge1": 0.14920395636285244, "rouge2": 0.05130151404400551, "rougeL": 0.11051783653694662, "avg_rouge": 0.10367443564793487 }, "prompt_5": { "rouge1": 0.15589644830480023, "rouge2": 0.05455147589607714, "rougeL": 0.11609765547693254, "avg_rouge": 0.10884852655926998 } }, "dialogsum": { "prompt_1": { "rouge1": 0.14162805950851104, "rouge2": 0.04235642933080281, "rougeL": 0.1066371383355487, "avg_rouge": 0.09687387572495419 }, "prompt_2": { "rouge1": 0.1558764099855247, "rouge2": 0.049274125590898915, "rougeL": 0.11990476794508834, "avg_rouge": 0.10835176784050399 }, "prompt_3": { "rouge1": 0.1531183009750366, "rouge2": 0.04781107644638046, "rougeL": 0.11863516112049498, "avg_rouge": 0.106521512847304 }, "prompt_4": { "rouge1": 0.13614545983306917, "rouge2": 0.03925473875709877, "rougeL": 0.10194629307087555, "avg_rouge": 0.09244883055368115 }, "prompt_5": { "rouge1": 0.15729831585992968, "rouge2": 0.03912829377948497, "rougeL": 0.11681467796576478, "avg_rouge": 0.10441376253505981 } }, "sst2": { "prompt_1": { "accuracy": 0.8784403669724771 }, "prompt_2": { "accuracy": 0.8646788990825688 }, "prompt_3": { "accuracy": 0.8772935779816514 }, "prompt_4": { "accuracy": 0.8589449541284404 }, "prompt_5": { "accuracy": 0.8646788990825688 } }, "cola": { "prompt_1": { "accuracy": 0.5704697986577181 }, "prompt_2": { "accuracy": 0.5819750719079578 }, "prompt_3": { "accuracy": 0.6059443911792906 }, "prompt_4": { "accuracy": 0.5781399808245445 }, "prompt_5": { "accuracy": 0.5685522531160115 } }, "qqp": { "prompt_1": { "accuracy": 0.507 }, "prompt_2": { "accuracy": 0.496 }, "prompt_3": { "accuracy": 0.4815 }, "prompt_4": { "accuracy": 0.4995 }, "prompt_5": { "accuracy": 0.504 } }, "mnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.4 }, "prompt_4": { "accuracy": 0.4 }, "prompt_5": { "accuracy": 0.5 } }, "qnli": { "prompt_1": { "accuracy": 0.8 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.3 }, "prompt_5": { "accuracy": 0.5 } }, "rte": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.4 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.6 } }, "mrpc": { "prompt_1": { "accuracy": 0.2 }, "prompt_2": { "accuracy": 0.3 }, "prompt_3": { "accuracy": 0.3 }, "prompt_4": { "accuracy": 0.5 }, "prompt_5": { "accuracy": 0.6 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } }, "qwen_1_5_7b": { "model_size": "7B", "model_link": "https://huggingface.co/Qwen/Qwen1.5-7B", "zero_shot": { "cross_xquad": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cross_logiqa": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sg_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cn_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "us_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ph_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sing2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "indommlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_ind2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_vie2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zho2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "flores_zsm2eng": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c_eval_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cmmlu_full": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "zbench": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ind_emotion": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "ocnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "c3": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dream": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "samsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "dialogsum": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "sst2": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "cola": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qqp": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "qnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "wnli": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "rte": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 }, "mrpc": { "prompt_1": -1, "prompt_2": -1, "prompt_3": -1, "prompt_4": -1, "prompt_5": -1 } }, "five_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9334033613445378, "language_acc": { "Spanish": 0.9252100840336135, "English": 0.9445378151260504, "Chinese": 0.9361344537815126, "Vietnamese": 0.9277310924369748 }, "consistency_score_2": 0.9215686274509806, "consistency_score_3": 0.8852941176470589, "consistency_score_4": 0.8596638655462185, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9378151260504202, "Spanish,Chinese": 0.9134453781512605, "Spanish,Vietnamese": 0.9168067226890756, "English,Chinese": 0.9252100840336135, "English,Vietnamese": 0.9243697478991597, "Chinese,Vietnamese": 0.9117647058823529 }, "3_combine": { "Spanish,English,Chinese": 0.8915966386554622, "Spanish,English,Vietnamese": 0.8907563025210085, "Spanish,Chinese,Vietnamese": 0.8739495798319328, "English,Chinese,Vietnamese": 0.8848739495798319 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8596638655462185 } }, "AC3_2": 0.9274482415064308, "AC3_3": 0.908712432595497, "AC3_4": 0.8950173531127907 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.5504761904761905, "language_acc": { "Filipino": 0.4, "Vietnamese": 0.52, "Chinese": 0.64, "Spanish": 0.6, "Malay": 0.49333333333333335, "Indonesian": 0.4866666666666667, "English": 0.7133333333333334 }, "consistency_score_2": 0.5803174603174602, "consistency_score_3": 0.4139047619047619, "consistency_score_4": 0.3243809523809523, "consistency_score_5": 0.26920634920634917, "consistency_score_6": 0.23047619047619047, "consistency_score_7": 0.2, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.46, "Filipino,Chinese": 0.48, "Filipino,Spanish": 0.48, "Filipino,Malay": 0.5733333333333334, "Filipino,Indonesian": 0.5466666666666666, "Filipino,English": 0.5333333333333333, "Vietnamese,Chinese": 0.5066666666666667, "Vietnamese,Spanish": 0.6066666666666667, "Vietnamese,Malay": 0.52, "Vietnamese,Indonesian": 0.56, "Vietnamese,English": 0.6533333333333333, "Chinese,Spanish": 0.6533333333333333, "Chinese,Malay": 0.5133333333333333, "Chinese,Indonesian": 0.5866666666666667, "Chinese,English": 0.6866666666666666, "Spanish,Malay": 0.6, "Spanish,Indonesian": 0.6733333333333333, "Spanish,English": 0.7333333333333333, "Malay,Indonesian": 0.6133333333333333, "Malay,English": 0.5866666666666667, "Indonesian,English": 0.62 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.28, "Filipino,Vietnamese,Spanish": 0.32666666666666666, "Filipino,Vietnamese,Malay": 0.34, "Filipino,Vietnamese,Indonesian": 0.34, "Filipino,Vietnamese,English": 0.36, "Filipino,Chinese,Spanish": 0.36, "Filipino,Chinese,Malay": 0.3466666666666667, "Filipino,Chinese,Indonesian": 0.36666666666666664, "Filipino,Chinese,English": 0.4066666666666667, "Filipino,Spanish,Malay": 0.38, "Filipino,Spanish,Indonesian": 0.3933333333333333, "Filipino,Spanish,English": 0.3933333333333333, "Filipino,Malay,Indonesian": 0.42, "Filipino,Malay,English": 0.4, "Filipino,Indonesian,English": 0.3933333333333333, "Vietnamese,Chinese,Spanish": 0.41333333333333333, "Vietnamese,Chinese,Malay": 0.3333333333333333, "Vietnamese,Chinese,Indonesian": 0.38, "Vietnamese,Chinese,English": 0.44666666666666666, "Vietnamese,Spanish,Malay": 0.4266666666666667, "Vietnamese,Spanish,Indonesian": 0.46, "Vietnamese,Spanish,English": 0.52, "Vietnamese,Malay,Indonesian": 0.4, "Vietnamese,Malay,English": 0.4266666666666667, "Vietnamese,Indonesian,English": 0.4533333333333333, "Chinese,Spanish,Malay": 0.41333333333333333, "Chinese,Spanish,Indonesian": 0.4866666666666667, "Chinese,Spanish,English": 0.5466666666666666, "Chinese,Malay,Indonesian": 0.4, "Chinese,Malay,English": 0.43333333333333335, "Chinese,Indonesian,English": 0.4866666666666667, "Spanish,Malay,Indonesian": 0.48, "Spanish,Malay,English": 0.4866666666666667, "Spanish,Indonesian,English": 0.54, "Malay,Indonesian,English": 0.44666666666666666 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Malay": 0.2733333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.28, "Filipino,Vietnamese,Spanish,English": 0.29333333333333333, "Filipino,Vietnamese,Malay,Indonesian": 0.29333333333333333, "Filipino,Vietnamese,Malay,English": 0.3, "Filipino,Vietnamese,Indonesian,English": 0.2866666666666667, "Filipino,Chinese,Spanish,Malay": 0.2866666666666667, "Filipino,Chinese,Spanish,Indonesian": 0.3, "Filipino,Chinese,Spanish,English": 0.32666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.2866666666666667, "Filipino,Chinese,Malay,English": 0.30666666666666664, "Filipino,Chinese,Indonesian,English": 0.32666666666666666, "Filipino,Spanish,Malay,Indonesian": 0.3333333333333333, "Filipino,Spanish,Malay,English": 0.32666666666666666, "Filipino,Spanish,Indonesian,English": 0.3333333333333333, "Filipino,Malay,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Chinese,Spanish,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,Spanish,English": 0.38, "Vietnamese,Chinese,Malay,Indonesian": 0.3, "Vietnamese,Chinese,Malay,English": 0.32666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.35333333333333333, "Vietnamese,Spanish,Malay,Indonesian": 0.3466666666666667, "Vietnamese,Spanish,Malay,English": 0.38666666666666666, "Vietnamese,Spanish,Indonesian,English": 0.42, "Vietnamese,Malay,Indonesian,English": 0.36, "Chinese,Spanish,Malay,Indonesian": 0.35333333333333333, "Chinese,Spanish,Malay,English": 0.38, "Chinese,Spanish,Indonesian,English": 0.4266666666666667, "Chinese,Malay,Indonesian,English": 0.36, "Spanish,Malay,Indonesian,English": 0.4 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.24, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay,English": 0.24, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.24, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.26, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, "Filipino,Chinese,Spanish,Malay,English": 0.26666666666666666, "Filipino,Chinese,Spanish,Indonesian,English": 0.28, "Filipino,Chinese,Malay,Indonesian,English": 0.26666666666666666, "Filipino,Spanish,Malay,Indonesian,English": 0.29333333333333333, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, "Vietnamese,Chinese,Spanish,Malay,English": 0.3, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3333333333333333, "Vietnamese,Chinese,Malay,Indonesian,English": 0.29333333333333333, "Vietnamese,Spanish,Malay,Indonesian,English": 0.3333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.32666666666666666 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.22, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.24, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2 } }, "AC3_2": 0.5650030747341315, "AC3_3": 0.4725201508120582, "AC3_4": 0.4082129119415617, "AC3_5": 0.36158312119818753, "AC3_6": 0.3249152148248294, "AC3_7": 0.29340101518933104 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.4967532467532467, "language_acc": { "Indonesian": 0.5227272727272727, "English": 0.5397727272727273, "Filipino": 0.3522727272727273, "Spanish": 0.5170454545454546, "Chinese": 0.625, "Malay": 0.44886363636363635, "Vietnamese": 0.4715909090909091 }, "consistency_score_2": 0.5395021645021645, "consistency_score_3": 0.36071428571428565, "consistency_score_4": 0.26720779220779217, "consistency_score_5": 0.21022727272727273, "consistency_score_6": 0.17126623376623376, "consistency_score_7": 0.14204545454545456, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5681818181818182, "Indonesian,Filipino": 0.45454545454545453, "Indonesian,Spanish": 0.5738636363636364, "Indonesian,Chinese": 0.5227272727272727, "Indonesian,Malay": 0.5965909090909091, "Indonesian,Vietnamese": 0.5454545454545454, "English,Filipino": 0.3977272727272727, "English,Spanish": 0.6704545454545454, "English,Chinese": 0.6761363636363636, "English,Malay": 0.5454545454545454, "English,Vietnamese": 0.625, "Filipino,Spanish": 0.48295454545454547, "Filipino,Chinese": 0.42613636363636365, "Filipino,Malay": 0.4431818181818182, "Filipino,Vietnamese": 0.42045454545454547, "Spanish,Chinese": 0.5738636363636364, "Spanish,Malay": 0.5681818181818182, "Spanish,Vietnamese": 0.5568181818181818, "Chinese,Malay": 0.5284090909090909, "Chinese,Vietnamese": 0.5397727272727273, "Malay,Vietnamese": 0.6136363636363636 }, "3_combine": { "Indonesian,English,Filipino": 0.26704545454545453, "Indonesian,English,Spanish": 0.4318181818181818, "Indonesian,English,Chinese": 0.4090909090909091, "Indonesian,English,Malay": 0.39204545454545453, "Indonesian,English,Vietnamese": 0.4034090909090909, "Indonesian,Filipino,Spanish": 0.32386363636363635, "Indonesian,Filipino,Chinese": 0.26704545454545453, "Indonesian,Filipino,Malay": 0.3068181818181818, "Indonesian,Filipino,Vietnamese": 0.2840909090909091, "Indonesian,Spanish,Chinese": 0.375, "Indonesian,Spanish,Malay": 0.4147727272727273, "Indonesian,Spanish,Vietnamese": 0.3693181818181818, "Indonesian,Chinese,Malay": 0.375, "Indonesian,Chinese,Vietnamese": 0.3693181818181818, "Indonesian,Malay,Vietnamese": 0.42613636363636365, "English,Filipino,Spanish": 0.32954545454545453, "English,Filipino,Chinese": 0.3068181818181818, "English,Filipino,Malay": 0.2727272727272727, "English,Filipino,Vietnamese": 0.2727272727272727, "English,Spanish,Chinese": 0.4943181818181818, "English,Spanish,Malay": 0.42613636363636365, "English,Spanish,Vietnamese": 0.44886363636363635, "English,Chinese,Malay": 0.4147727272727273, "English,Chinese,Vietnamese": 0.45454545454545453, "English,Malay,Vietnamese": 0.4318181818181818, "Filipino,Spanish,Chinese": 0.30113636363636365, "Filipino,Spanish,Malay": 0.32386363636363635, "Filipino,Spanish,Vietnamese": 0.30113636363636365, "Filipino,Chinese,Malay": 0.2840909090909091, "Filipino,Chinese,Vietnamese": 0.26704545454545453, "Filipino,Malay,Vietnamese": 0.3125, "Spanish,Chinese,Malay": 0.39204545454545453, "Spanish,Chinese,Vietnamese": 0.3806818181818182, "Spanish,Malay,Vietnamese": 0.4147727272727273, "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.22727272727272727, "Indonesian,English,Filipino,Chinese": 0.2159090909090909, "Indonesian,English,Filipino,Malay": 0.2215909090909091, "Indonesian,English,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Chinese": 0.32954545454545453, "Indonesian,English,Spanish,Malay": 0.32386363636363635, "Indonesian,English,Spanish,Vietnamese": 0.3125, "Indonesian,English,Chinese,Malay": 0.3125, "Indonesian,English,Chinese,Vietnamese": 0.3068181818181818, "Indonesian,English,Malay,Vietnamese": 0.3352272727272727, "Indonesian,Filipino,Spanish,Chinese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Malay": 0.25, "Indonesian,Filipino,Spanish,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Chinese,Malay": 0.21022727272727273, "Indonesian,Filipino,Chinese,Vietnamese": 0.2159090909090909, "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Spanish,Chinese,Malay": 0.29545454545454547, "Indonesian,Spanish,Chinese,Vietnamese": 0.26704545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.3352272727272727, "Indonesian,Chinese,Malay,Vietnamese": 0.3068181818181818, "English,Filipino,Spanish,Chinese": 0.2556818181818182, "English,Filipino,Spanish,Malay": 0.23295454545454544, "English,Filipino,Spanish,Vietnamese": 0.23863636363636365, "English,Filipino,Chinese,Malay": 0.22727272727272727, "English,Filipino,Chinese,Vietnamese": 0.22727272727272727, "English,Filipino,Malay,Vietnamese": 0.22727272727272727, "English,Spanish,Chinese,Malay": 0.3465909090909091, "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, "English,Spanish,Malay,Vietnamese": 0.3465909090909091, "English,Chinese,Malay,Vietnamese": 0.32954545454545453, "Filipino,Spanish,Chinese,Malay": 0.2215909090909091, "Filipino,Spanish,Chinese,Vietnamese": 0.2215909090909091, "Filipino,Spanish,Malay,Vietnamese": 0.25, "Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, "Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.18181818181818182, "Indonesian,English,Filipino,Spanish,Malay": 0.1875, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Chinese,Malay": 0.1875, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.18181818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Chinese,Malay": 0.26704545454545453, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.25, "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, "Indonesian,English,Chinese,Malay,Vietnamese": 0.26704545454545453, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, "English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, "English,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "English,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "English,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "English,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 } }, "AC3_2": 0.517245939389236, "AC3_3": 0.41794233781029455, "AC3_4": 0.347495046344588, "AC3_5": 0.2954284521030827, "AC3_6": 0.25471430145624463, "AC3_7": 0.22091948708566592 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6699029126213593 } }, "cn_eval": { "prompt_1": { "accuracy": 0.8 } }, "us_eval": { "prompt_1": { "accuracy": 0.6915887850467289 } }, "ph_eval": { "prompt_1": { "accuracy": 0.41, "category_acc": { "brand": 0.5, "demographics": 0.0, "biology": 0.3, "history": 0.26666666666666666, "literature": 0.2, "politics": 0.5, "culture": 0.7, "film": 0.3, "law": 0.5, "geography": 0.7 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.324299850031996 } }, "indommlu": { "prompt_1": { "accuracy": 0.4615795446959076, "category_acc": { "History": 0.43373493975903615, "Geography": 0.463265306122449, "Lampungic": 0.2653061224489796, "Social science": 0.6828046744574291, "Balinese": 0.2823779193205945, "Makassarese": 0.3548387096774194, "Banjarese": 0.3819444444444444, "Chemistry": 0.3284671532846715, "Biology": 0.39644970414201186, "Science": 0.5882352941176471, "Christian religion": 0.5422885572139303, "Art": 0.5457570715474209, "Islam religion": 0.5305832147937412, "Hindu religion": 0.44666666666666666, "Madurese": 0.2745762711864407, "Sport": 0.5, "Indonesian language": 0.5438978829389788, "Physics": 0.4080808080808081, "Minangkabau culture": 0.36180904522613067, "Dayak language": 0.3486238532110092, "Sociology": 0.4717741935483871, "Economy": 0.4426229508196721, "Sundanese": 0.3526361279170268, "Javanese": 0.3034274193548387, "Civic education": 0.5565092989985694 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.358063538041651 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.30566338247234903 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.25377859749436343 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.3455666218492509 } }, "mmlu": { "prompt_1": { "accuracy": 0.5997666277712952 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.5871290668573471, "category_acc": { "high_school_european_history": 0.725609756097561, "business_ethics": 0.6767676767676768, "clinical_knowledge": 0.6818181818181818, "medical_genetics": 0.6565656565656566, "high_school_us_history": 0.7684729064039408, "high_school_physics": 0.4, "high_school_world_history": 0.7838983050847458, "virology": 0.4666666666666667, "high_school_microeconomics": 0.6371308016877637, "econometrics": 0.415929203539823, "college_computer_science": 0.494949494949495, "high_school_biology": 0.7119741100323624, "abstract_algebra": 0.3434343434343434, "professional_accounting": 0.45907473309608543, "philosophy": 0.6709677419354839, "professional_medicine": 0.6014760147601476, "nutrition": 0.6688524590163935, "global_facts": 0.40404040404040403, "machine_learning": 0.43243243243243246, "security_studies": 0.7131147540983607, "public_relations": 0.5412844036697247, "professional_psychology": 0.5793780687397708, "prehistory": 0.6934984520123839, "anatomy": 0.5223880597014925, "human_sexuality": 0.7153846153846154, "college_medicine": 0.622093023255814, "high_school_government_and_politics": 0.8177083333333334, "college_chemistry": 0.40404040404040403, "logical_fallacies": 0.6481481481481481, "high_school_geography": 0.7817258883248731, "elementary_mathematics": 0.46949602122015915, "human_aging": 0.6306306306306306, "college_mathematics": 0.3939393939393939, "high_school_psychology": 0.7959558823529411, "formal_logic": 0.376, "high_school_statistics": 0.4558139534883721, "international_law": 0.7583333333333333, "high_school_mathematics": 0.3271375464684015, "high_school_computer_science": 0.6464646464646465, "conceptual_physics": 0.5213675213675214, "miscellaneous": 0.7851662404092071, "high_school_chemistry": 0.5693069306930693, "marketing": 0.8369098712446352, "professional_law": 0.42204827136333983, "management": 0.803921568627451, "college_physics": 0.43564356435643564, "jurisprudence": 0.7102803738317757, "world_religions": 0.7529411764705882, "sociology": 0.79, "us_foreign_policy": 0.8383838383838383, "high_school_macroeconomics": 0.5655526992287918, "computer_security": 0.7373737373737373, "moral_scenarios": 0.2841163310961969, "moral_disputes": 0.6434782608695652, "electrical_engineering": 0.5277777777777778, "astronomy": 0.6225165562913907, "college_biology": 0.6433566433566433 } } }, "c_eval": { "prompt_1": { "accuracy": 0.6946508172362555 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.7235367372353674, "category_acc": { "computer_network": 0.6666666666666666, "operating_system": 0.5833333333333334, "computer_architecture": 0.6153846153846154, "college_programming": 0.6666666666666666, "college_physics": 0.625, "college_chemistry": 0.5517241379310345, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.5217391304347826, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.5238095238095238, "metrology_engineer": 0.7241379310344828, "high_school_mathematics": 0.5217391304347826, "high_school_physics": 0.8333333333333334, "high_school_chemistry": 0.7083333333333334, "high_school_biology": 0.9166666666666666, "middle_school_mathematics": 0.7083333333333334, "middle_school_biology": 0.9615384615384616, "middle_school_physics": 0.875, "middle_school_chemistry": 0.96, "veterinary_medicine": 0.8214285714285714, "college_economics": 0.6, "business_administration": 0.7631578947368421, "marxism": 0.875, "mao_zedong_thought": 0.896551724137931, "education_science": 0.7941176470588235, "teacher_qualification": 0.8571428571428571, "high_school_politics": 0.9583333333333334, "high_school_geography": 0.9166666666666666, "middle_school_politics": 0.9615384615384616, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.8928571428571429, "ideological_and_moral_cultivation": 1.0, "logic": 0.5185185185185185, "law": 0.6896551724137931, "chinese_language_and_literature": 0.6785714285714286, "art_studies": 0.7105263157894737, "professional_tour_guide": 0.7941176470588235, "legal_professional": 0.5357142857142857, "high_school_chinese": 0.8333333333333334, "high_school_history": 0.92, "middle_school_history": 0.9629629629629629, "civil_servant": 0.6346153846153846, "sports_science": 0.625, "plant_protection": 0.8888888888888888, "basic_medicine": 0.8333333333333334, "clinical_medicine": 0.6666666666666666, "urban_and_rural_planner": 0.6470588235294118, "accountant": 0.7037037037037037, "fire_engineer": 0.6388888888888888, "environmental_impact_assessment_engineer": 0.6666666666666666, "tax_accountant": 0.6666666666666666, "physician": 0.6851851851851852 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.7204301075268817 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.7157658435503367, "category_acc": { "agronomy": 0.5976331360946746, "anatomy": 0.7027027027027027, "ancient_chinese": 0.42073170731707316, "arts": 0.88125, "astronomy": 0.4, "business_ethics": 0.6411483253588517, "chinese_civil_service_exam": 0.675, "chinese_driving_rule": 0.9007633587786259, "chinese_food_culture": 0.6691176470588235, "chinese_foreign_policy": 0.8504672897196262, "chinese_history": 0.9411764705882353, "chinese_literature": 0.5980392156862745, "chinese_teacher_qualification": 0.8938547486033519, "clinical_knowledge": 0.6919831223628692, "college_actuarial_science": 0.32075471698113206, "college_education": 0.822429906542056, "college_engineering_hydrology": 0.5566037735849056, "college_law": 0.6759259259259259, "college_mathematics": 0.4, "college_medical_statistics": 0.6132075471698113, "college_medicine": 0.7509157509157509, "computer_science": 0.7401960784313726, "computer_security": 0.8362573099415205, "conceptual_physics": 0.8503401360544217, "construction_project_management": 0.60431654676259, "economics": 0.6792452830188679, "education": 0.7607361963190185, "electrical_engineering": 0.7383720930232558, "elementary_chinese": 0.7222222222222222, "elementary_commonsense": 0.696969696969697, "elementary_information_and_technology": 0.8823529411764706, "elementary_mathematics": 0.5304347826086957, "ethnology": 0.6518518518518519, "food_science": 0.6083916083916084, "genetics": 0.5568181818181818, "global_facts": 0.7449664429530202, "high_school_biology": 0.8994082840236687, "high_school_chemistry": 0.7727272727272727, "high_school_geography": 0.7966101694915254, "high_school_mathematics": 0.5, "high_school_physics": 0.6818181818181818, "high_school_politics": 0.8321678321678322, "human_sexuality": 0.6111111111111112, "international_law": 0.6162162162162163, "journalism": 0.6337209302325582, "jurisprudence": 0.7712895377128953, "legal_and_moral_basis": 0.9485981308411215, "logical": 0.6016260162601627, "machine_learning": 0.6229508196721312, "management": 0.8333333333333334, "marketing": 0.7444444444444445, "marxist_theory": 0.9153439153439153, "modern_chinese": 0.5689655172413793, "nutrition": 0.7379310344827587, "philosophy": 0.7047619047619048, "professional_accounting": 0.8228571428571428, "professional_law": 0.6350710900473934, "professional_medicine": 0.6382978723404256, "professional_psychology": 0.8318965517241379, "public_relations": 0.6896551724137931, "security_study": 0.7851851851851852, "sociology": 0.6769911504424779, "sports_science": 0.6909090909090909, "traditional_chinese_medicine": 0.7675675675675676, "virology": 0.7337278106508875, "world_history": 0.9130434782608695, "world_religions": 0.70625 } } }, "zbench": { "prompt_1": { "accuracy": 0.45454545454545453 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.634090909090909 } }, "ocnli": { "prompt_1": { "accuracy": 0.615593220338983 } }, "c3": { "prompt_1": { "accuracy": 0.8975317875841436 } }, "dream": { "prompt_1": { "accuracy": 0.8990690837824595 } }, "samsum": { "prompt_1": { "rouge1": 0.40619506364326374, "rouge2": 0.17465809475148847, "rougeL": 0.3178215644999177, "avg_rouge": 0.29955824096489 } }, "dialogsum": { "prompt_1": { "rouge1": 0.3588273806142116, "rouge2": 0.1380414207543411, "rougeL": 0.27862569710756147, "avg_rouge": 0.25849816615870475 } }, "sst2": { "prompt_1": { "accuracy": 0.9438073394495413 } }, "cola": { "prompt_1": { "accuracy": 0.7996164908916586 } }, "qqp": { "prompt_1": { "accuracy": 0.774 } }, "mnli": { "prompt_1": { "accuracy": 0.8 } }, "qnli": { "prompt_1": { "accuracy": 0.8 } }, "wnli": { "prompt_1": { "accuracy": 0.5 } }, "rte": { "prompt_1": { "accuracy": 1.0 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 } } } }, "qwen_1_5_7b_chat": { "model_size": "7B", "model_link": "https://huggingface.co/Qwen/Qwen1.5-7B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": { "overall_acc": 0.9256302521008403, "language_acc": { "Spanish": 0.9243697478991597, "English": 0.9369747899159664, "Chinese": 0.9252100840336135, "Vietnamese": 0.9159663865546218 }, "consistency_score_2": 0.9183473389355744, "consistency_score_3": 0.8817226890756302, "consistency_score_4": 0.8588235294117647, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9285714285714286, "Spanish,Chinese": 0.9084033613445378, "Spanish,Vietnamese": 0.9176470588235294, "English,Chinese": 0.926890756302521, "English,Vietnamese": 0.9226890756302522, "Chinese,Vietnamese": 0.9058823529411765 }, "3_combine": { "Spanish,English,Chinese": 0.8857142857142857, "Spanish,English,Vietnamese": 0.8899159663865546, "Spanish,Chinese,Vietnamese": 0.8697478991596639, "English,Chinese,Vietnamese": 0.8815126050420168 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8588235294117647 } }, "AC3_2": 0.9219744132912145, "AC3_3": 0.9031431286417014, "AC3_4": 0.8909763292621103 }, "prompt_2": { "overall_acc": 0.9292016806722688, "language_acc": { "Spanish": 0.9319327731092437, "English": 0.9436974789915966, "Chinese": 0.9294117647058824, "Vietnamese": 0.9117647058823529 }, "consistency_score_2": 0.9148459383753503, "consistency_score_3": 0.8760504201680672, "consistency_score_4": 0.8478991596638655, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9277310924369748, "Spanish,Chinese": 0.9134453781512605, "Spanish,Vietnamese": 0.9117647058823529, "English,Chinese": 0.9260504201680673, "English,Vietnamese": 0.9109243697478991, "Chinese,Vietnamese": 0.8991596638655462 }, "3_combine": { "Spanish,English,Chinese": 0.8857142857142857, "Spanish,English,Vietnamese": 0.8789915966386554, "Spanish,Chinese,Vietnamese": 0.8672268907563025, "English,Chinese,Vietnamese": 0.8722689075630252 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8478991596638655 } }, "AC3_2": 0.9219679304024787, "AC3_3": 0.9018435955286216, "AC3_4": 0.8866906213458434 }, "prompt_3": { "overall_acc": 0.917016806722689, "language_acc": { "Spanish": 0.9134453781512605, "English": 0.9327731092436975, "Chinese": 0.915126050420168, "Vietnamese": 0.9067226890756303 }, "consistency_score_2": 0.9012605042016807, "consistency_score_3": 0.8581932773109244, "consistency_score_4": 0.826890756302521, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9134453781512605, "Spanish,Chinese": 0.8865546218487395, "Spanish,Vietnamese": 0.8966386554621849, "English,Chinese": 0.9100840336134454, "English,Vietnamese": 0.9092436974789916, "Chinese,Vietnamese": 0.8915966386554622 }, "3_combine": { "Spanish,English,Chinese": 0.8596638655462185, "Spanish,English,Vietnamese": 0.8647058823529412, "Spanish,Chinese,Vietnamese": 0.8453781512605042, "English,Chinese,Vietnamese": 0.8630252100840337 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.826890756302521 } }, "AC3_2": 0.909070387203692, "AC3_3": 0.8866304509470218, "AC3_4": 0.8696249008682165 }, "prompt_4": { "overall_acc": 0.9235294117647058, "language_acc": { "Spanish": 0.9277310924369748, "English": 0.9336134453781513, "Chinese": 0.9210084033613445, "Vietnamese": 0.9117647058823529 }, "consistency_score_2": 0.9123249299719888, "consistency_score_3": 0.873109243697479, "consistency_score_4": 0.8470588235294118, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9302521008403362, "Spanish,Chinese": 0.9050420168067227, "Spanish,Vietnamese": 0.9176470588235294, "English,Chinese": 0.9126050420168067, "English,Vietnamese": 0.9134453781512605, "Chinese,Vietnamese": 0.8949579831932774 }, "3_combine": { "Spanish,English,Chinese": 0.8756302521008403, "Spanish,English,Vietnamese": 0.8848739495798319, "Spanish,Chinese,Vietnamese": 0.8647058823529412, "English,Chinese,Vietnamese": 0.8672268907563025 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8470588235294118 } }, "AC3_2": 0.9178929795403732, "AC3_3": 0.8976118416932849, "AC3_4": 0.8836427593794464 }, "prompt_5": { "overall_acc": 0.9077731092436975, "language_acc": { "Spanish": 0.907563025210084, "English": 0.915126050420168, "Chinese": 0.9142857142857143, "Vietnamese": 0.8941176470588236 }, "consistency_score_2": 0.892577030812325, "consistency_score_3": 0.8441176470588235, "consistency_score_4": 0.8117647058823529, "detailed_consistency_score": { "2_combine": { "Spanish,English": 0.9084033613445378, "Spanish,Chinese": 0.8899159663865546, "Spanish,Vietnamese": 0.8873949579831932, "English,Chinese": 0.8974789915966387, "English,Vietnamese": 0.8915966386554622, "Chinese,Vietnamese": 0.880672268907563 }, "3_combine": { "Spanish,English,Chinese": 0.8529411764705882, "Spanish,English,Vietnamese": 0.8478991596638655, "Spanish,Chinese,Vietnamese": 0.8344537815126051, "English,Chinese,Vietnamese": 0.8411764705882353 }, "4_combine": { "Spanish,English,Chinese,Vietnamese": 0.8117647058823529 } }, "AC3_2": 0.9001109377865719, "AC3_3": 0.8747889081983395, "AC3_4": 0.8570886484822626 } }, "cross_mmlu": { "prompt_1": { "overall_acc": 0.5399999999999999, "language_acc": { "Filipino": 0.4533333333333333, "Vietnamese": 0.5133333333333333, "Chinese": 0.62, "Spanish": 0.5733333333333334, "Malay": 0.44666666666666666, "Indonesian": 0.48, "English": 0.6933333333333334 }, "consistency_score_2": 0.5114285714285715, "consistency_score_3": 0.3398095238095238, "consistency_score_4": 0.252952380952381, "consistency_score_5": 0.19904761904761903, "consistency_score_6": 0.16190476190476186, "consistency_score_7": 0.13333333333333333, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.41333333333333333, "Filipino,Chinese": 0.4533333333333333, "Filipino,Spanish": 0.47333333333333333, "Filipino,Malay": 0.46, "Filipino,Indonesian": 0.4533333333333333, "Filipino,English": 0.5133333333333333, "Vietnamese,Chinese": 0.4866666666666667, "Vietnamese,Spanish": 0.5333333333333333, "Vietnamese,Malay": 0.48, "Vietnamese,Indonesian": 0.49333333333333335, "Vietnamese,English": 0.5466666666666666, "Chinese,Spanish": 0.56, "Chinese,Malay": 0.47333333333333333, "Chinese,Indonesian": 0.5066666666666667, "Chinese,English": 0.64, "Spanish,Malay": 0.44, "Spanish,Indonesian": 0.56, "Spanish,English": 0.6466666666666666, "Malay,Indonesian": 0.5533333333333333, "Malay,English": 0.47333333333333333, "Indonesian,English": 0.58 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.26, "Filipino,Vietnamese,Spanish": 0.31333333333333335, "Filipino,Vietnamese,Malay": 0.2733333333333333, "Filipino,Vietnamese,Indonesian": 0.2733333333333333, "Filipino,Vietnamese,English": 0.30666666666666664, "Filipino,Chinese,Spanish": 0.32, "Filipino,Chinese,Malay": 0.2866666666666667, "Filipino,Chinese,Indonesian": 0.2866666666666667, "Filipino,Chinese,English": 0.36, "Filipino,Spanish,Malay": 0.29333333333333333, "Filipino,Spanish,Indonesian": 0.32, "Filipino,Spanish,English": 0.37333333333333335, "Filipino,Malay,Indonesian": 0.32, "Filipino,Malay,English": 0.32, "Filipino,Indonesian,English": 0.3466666666666667, "Vietnamese,Chinese,Spanish": 0.3466666666666667, "Vietnamese,Chinese,Malay": 0.2866666666666667, "Vietnamese,Chinese,Indonesian": 0.32, "Vietnamese,Chinese,English": 0.4, "Vietnamese,Spanish,Malay": 0.32666666666666666, "Vietnamese,Spanish,Indonesian": 0.36, "Vietnamese,Spanish,English": 0.41333333333333333, "Vietnamese,Malay,Indonesian": 0.3466666666666667, "Vietnamese,Malay,English": 0.34, "Vietnamese,Indonesian,English": 0.38666666666666666, "Chinese,Spanish,Malay": 0.31333333333333335, "Chinese,Spanish,Indonesian": 0.37333333333333335, "Chinese,Spanish,English": 0.4666666666666667, "Chinese,Malay,Indonesian": 0.3333333333333333, "Chinese,Malay,English": 0.34, "Chinese,Indonesian,English": 0.4, "Spanish,Malay,Indonesian": 0.32666666666666666, "Spanish,Malay,English": 0.35333333333333333, "Spanish,Indonesian,English": 0.44666666666666666, "Malay,Indonesian,English": 0.36 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, "Filipino,Vietnamese,Spanish,Malay": 0.24, "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Vietnamese,Spanish,English": 0.26, "Filipino,Vietnamese,Malay,Indonesian": 0.23333333333333334, "Filipino,Vietnamese,Malay,English": 0.23333333333333334, "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, "Filipino,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, "Filipino,Chinese,Spanish,English": 0.2733333333333333, "Filipino,Chinese,Malay,Indonesian": 0.21333333333333335, "Filipino,Chinese,Malay,English": 0.22666666666666666, "Filipino,Chinese,Indonesian,English": 0.24666666666666667, "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, "Filipino,Spanish,Malay,English": 0.25333333333333335, "Filipino,Spanish,Indonesian,English": 0.2866666666666667, "Filipino,Malay,Indonesian,English": 0.26666666666666666, "Vietnamese,Chinese,Spanish,Malay": 0.24, "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,Spanish,English": 0.31333333333333335, "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Malay,English": 0.24666666666666667, "Vietnamese,Chinese,Indonesian,English": 0.2866666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, "Vietnamese,Spanish,Malay,English": 0.28, "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, "Vietnamese,Malay,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, "Chinese,Spanish,Malay,English": 0.28, "Chinese,Spanish,Indonesian,English": 0.3333333333333333, "Chinese,Malay,Indonesian,English": 0.26, "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Spanish,English": 0.19333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,English": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, "Filipino,Vietnamese,Malay,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, "Filipino,Chinese,Spanish,Malay,English": 0.18666666666666668, "Filipino,Chinese,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Chinese,Malay,Indonesian,English": 0.18666666666666668, "Filipino,Spanish,Malay,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, "Vietnamese,Chinese,Spanish,Malay,English": 0.22, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.25333333333333335, "Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, "Vietnamese,Spanish,Malay,Indonesian,English": 0.24, "Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.18, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 } }, "AC3_2": 0.5253260869065587, "AC3_3": 0.4171292487077305, "AC3_4": 0.3445207782414429, "AC3_5": 0.2908762886204355, "AC3_6": 0.24911804609747973, "AC3_7": 0.2138613861068523 }, "prompt_2": { "overall_acc": 0.5152380952380952, "language_acc": { "Filipino": 0.3933333333333333, "Vietnamese": 0.46, "Chinese": 0.6266666666666667, "Spanish": 0.6, "Malay": 0.4, "Indonesian": 0.49333333333333335, "English": 0.6333333333333333 }, "consistency_score_2": 0.4695238095238095, "consistency_score_3": 0.28476190476190477, "consistency_score_4": 0.19238095238095235, "consistency_score_5": 0.13555555555555557, "consistency_score_6": 0.09619047619047619, "consistency_score_7": 0.06666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.42, "Filipino,Chinese": 0.4533333333333333, "Filipino,Spanish": 0.41333333333333333, "Filipino,Malay": 0.42, "Filipino,Indonesian": 0.4, "Filipino,English": 0.4066666666666667, "Vietnamese,Chinese": 0.44666666666666666, "Vietnamese,Spanish": 0.46, "Vietnamese,Malay": 0.49333333333333335, "Vietnamese,Indonesian": 0.47333333333333333, "Vietnamese,English": 0.47333333333333333, "Chinese,Spanish": 0.5666666666666667, "Chinese,Malay": 0.46, "Chinese,Indonesian": 0.43333333333333335, "Chinese,English": 0.5533333333333333, "Spanish,Malay": 0.4666666666666667, "Spanish,Indonesian": 0.49333333333333335, "Spanish,English": 0.5866666666666667, "Malay,Indonesian": 0.5133333333333333, "Malay,English": 0.4533333333333333, "Indonesian,English": 0.47333333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.22666666666666666, "Filipino,Vietnamese,Spanish": 0.24666666666666667, "Filipino,Vietnamese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Indonesian": 0.24666666666666667, "Filipino,Vietnamese,English": 0.25333333333333335, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.26, "Filipino,Chinese,Indonesian": 0.23333333333333334, "Filipino,Chinese,English": 0.28, "Filipino,Spanish,Malay": 0.23333333333333334, "Filipino,Spanish,Indonesian": 0.22666666666666666, "Filipino,Spanish,English": 0.28, "Filipino,Malay,Indonesian": 0.26666666666666666, "Filipino,Malay,English": 0.22666666666666666, "Filipino,Indonesian,English": 0.22666666666666666, "Vietnamese,Chinese,Spanish": 0.3, "Vietnamese,Chinese,Malay": 0.26666666666666666, "Vietnamese,Chinese,Indonesian": 0.26, "Vietnamese,Chinese,English": 0.3333333333333333, "Vietnamese,Spanish,Malay": 0.30666666666666664, "Vietnamese,Spanish,Indonesian": 0.2866666666666667, "Vietnamese,Spanish,English": 0.32666666666666666, "Vietnamese,Malay,Indonesian": 0.32, "Vietnamese,Malay,English": 0.29333333333333333, "Vietnamese,Indonesian,English": 0.28, "Chinese,Spanish,Malay": 0.32, "Chinese,Spanish,Indonesian": 0.31333333333333335, "Chinese,Spanish,English": 0.41333333333333333, "Chinese,Malay,Indonesian": 0.30666666666666664, "Chinese,Malay,English": 0.30666666666666664, "Chinese,Indonesian,English": 0.32, "Spanish,Malay,Indonesian": 0.30666666666666664, "Spanish,Malay,English": 0.31333333333333335, "Spanish,Indonesian,English": 0.3466666666666667, "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.16666666666666666, "Filipino,Vietnamese,Chinese,Malay": 0.15333333333333332, "Filipino,Vietnamese,Chinese,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,English": 0.18, "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Indonesian": 0.15333333333333332, "Filipino,Vietnamese,Spanish,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Malay,English": 0.16, "Filipino,Vietnamese,Indonesian,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Malay": 0.16666666666666666, "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, "Filipino,Chinese,Spanish,English": 0.21333333333333335, "Filipino,Chinese,Malay,Indonesian": 0.18, "Filipino,Chinese,Malay,English": 0.18, "Filipino,Chinese,Indonesian,English": 0.18, "Filipino,Spanish,Malay,Indonesian": 0.16, "Filipino,Spanish,Malay,English": 0.16, "Filipino,Spanish,Indonesian,English": 0.16, "Filipino,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Spanish,English": 0.26, "Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, "Vietnamese,Chinese,Malay,English": 0.21333333333333335, "Vietnamese,Chinese,Indonesian,English": 0.21333333333333335, "Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Spanish,Malay,English": 0.22, "Vietnamese,Spanish,Indonesian,English": 0.21333333333333335, "Vietnamese,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Chinese,Spanish,Malay,English": 0.24666666666666667, "Chinese,Spanish,Indonesian,English": 0.26, "Chinese,Malay,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian,English": 0.24 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.1, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Malay,English": 0.12, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Spanish,Malay,English": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.10666666666666667, "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Chinese,Spanish,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Malay,Indonesian,English": 0.14, "Filipino,Spanish,Malay,Indonesian,English": 0.12, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.1, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.08666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 } }, "AC3_2": 0.49131988573807855, "AC3_3": 0.3668004534688892, "AC3_4": 0.28015638014370664, "AC3_5": 0.2146406503735227, "AC3_6": 0.16211541311693634, "AC3_7": 0.1180578286758384 }, "prompt_3": { "overall_acc": 0.4752380952380952, "language_acc": { "Filipino": 0.35333333333333333, "Vietnamese": 0.46, "Chinese": 0.6133333333333333, "Spanish": 0.5133333333333333, "Malay": 0.38, "Indonesian": 0.46, "English": 0.5466666666666666 }, "consistency_score_2": 0.4641269841269841, "consistency_score_3": 0.2773333333333333, "consistency_score_4": 0.18647619047619043, "consistency_score_5": 0.1346031746031746, "consistency_score_6": 0.10380952380952381, "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.38666666666666666, "Filipino,Chinese": 0.42, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.36666666666666664, "Filipino,Indonesian": 0.38666666666666666, "Filipino,English": 0.43333333333333335, "Vietnamese,Chinese": 0.4266666666666667, "Vietnamese,Spanish": 0.47333333333333333, "Vietnamese,Malay": 0.4266666666666667, "Vietnamese,Indonesian": 0.4266666666666667, "Vietnamese,English": 0.4866666666666667, "Chinese,Spanish": 0.5466666666666666, "Chinese,Malay": 0.38666666666666666, "Chinese,Indonesian": 0.52, "Chinese,English": 0.56, "Spanish,Malay": 0.46, "Spanish,Indonesian": 0.54, "Spanish,English": 0.5666666666666667, "Malay,Indonesian": 0.5, "Malay,English": 0.4666666666666667, "Indonesian,English": 0.5133333333333333 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.21333333333333335, "Filipino,Vietnamese,Spanish": 0.23333333333333334, "Filipino,Vietnamese,Malay": 0.18666666666666668, "Filipino,Vietnamese,Indonesian": 0.20666666666666667, "Filipino,Vietnamese,English": 0.24666666666666667, "Filipino,Chinese,Spanish": 0.3, "Filipino,Chinese,Malay": 0.18666666666666668, "Filipino,Chinese,Indonesian": 0.25333333333333335, "Filipino,Chinese,English": 0.3, "Filipino,Spanish,Malay": 0.22, "Filipino,Spanish,Indonesian": 0.2733333333333333, "Filipino,Spanish,English": 0.29333333333333333, "Filipino,Malay,Indonesian": 0.21333333333333335, "Filipino,Malay,English": 0.20666666666666667, "Filipino,Indonesian,English": 0.24666666666666667, "Vietnamese,Chinese,Spanish": 0.28, "Vietnamese,Chinese,Malay": 0.22666666666666666, "Vietnamese,Chinese,Indonesian": 0.26666666666666666, "Vietnamese,Chinese,English": 0.32, "Vietnamese,Spanish,Malay": 0.2866666666666667, "Vietnamese,Spanish,Indonesian": 0.29333333333333333, "Vietnamese,Spanish,English": 0.32666666666666666, "Vietnamese,Malay,Indonesian": 0.28, "Vietnamese,Malay,English": 0.26666666666666666, "Vietnamese,Indonesian,English": 0.3, "Chinese,Spanish,Malay": 0.2733333333333333, "Chinese,Spanish,Indonesian": 0.3466666666666667, "Chinese,Spanish,English": 0.4, "Chinese,Malay,Indonesian": 0.29333333333333333, "Chinese,Malay,English": 0.2733333333333333, "Chinese,Indonesian,English": 0.3466666666666667, "Spanish,Malay,Indonesian": 0.31333333333333335, "Spanish,Malay,English": 0.32, "Spanish,Indonesian,English": 0.38, "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.16, "Filipino,Vietnamese,Chinese,Malay": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,English": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Malay": 0.13333333333333333, "Filipino,Vietnamese,Spanish,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Spanish,English": 0.18, "Filipino,Vietnamese,Malay,Indonesian": 0.14, "Filipino,Vietnamese,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian": 0.2, "Filipino,Chinese,Spanish,English": 0.23333333333333334, "Filipino,Chinese,Malay,Indonesian": 0.15333333333333332, "Filipino,Chinese,Malay,English": 0.15333333333333332, "Filipino,Chinese,Indonesian,English": 0.2, "Filipino,Spanish,Malay,Indonesian": 0.16, "Filipino,Spanish,Malay,English": 0.16666666666666666, "Filipino,Spanish,Indonesian,English": 0.2, "Filipino,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.2, "Vietnamese,Chinese,Spanish,English": 0.22666666666666666, "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Malay,English": 0.17333333333333334, "Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, "Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Spanish,Malay,English": 0.2, "Vietnamese,Spanish,Indonesian,English": 0.23333333333333334, "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, "Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, "Chinese,Spanish,Malay,English": 0.22666666666666666, "Chinese,Spanish,Indonesian,English": 0.2733333333333333, "Chinese,Malay,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.1, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.10666666666666667, "Filipino,Vietnamese,Spanish,Malay,English": 0.1, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Malay,Indonesian,English": 0.10666666666666667, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,English": 0.14, "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.14, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,English": 0.14, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.09333333333333334, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, "AC3_2": 0.4696168267415549, "AC3_3": 0.35026406812185384, "AC3_4": 0.26785152283695113, "AC3_5": 0.20978756100771556, "AC3_6": 0.1703978696447582, "AC3_7": 0.14659887003040753 }, "prompt_4": { "overall_acc": 0.5304761904761904, "language_acc": { "Filipino": 0.4066666666666667, "Vietnamese": 0.5, "Chinese": 0.64, "Spanish": 0.5533333333333333, "Malay": 0.4266666666666667, "Indonesian": 0.5, "English": 0.6866666666666666 }, "consistency_score_2": 0.49777777777777776, "consistency_score_3": 0.3182857142857144, "consistency_score_4": 0.22514285714285714, "consistency_score_5": 0.1688888888888889, "consistency_score_6": 0.13238095238095238, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4, "Filipino,Chinese": 0.5, "Filipino,Spanish": 0.4533333333333333, "Filipino,Malay": 0.41333333333333333, "Filipino,Indonesian": 0.4533333333333333, "Filipino,English": 0.44, "Vietnamese,Chinese": 0.52, "Vietnamese,Spanish": 0.49333333333333335, "Vietnamese,Malay": 0.43333333333333335, "Vietnamese,Indonesian": 0.5133333333333333, "Vietnamese,English": 0.5066666666666667, "Chinese,Spanish": 0.5866666666666667, "Chinese,Malay": 0.4266666666666667, "Chinese,Indonesian": 0.5333333333333333, "Chinese,English": 0.6133333333333333, "Spanish,Malay": 0.5, "Spanish,Indonesian": 0.5533333333333333, "Spanish,English": 0.64, "Malay,Indonesian": 0.4866666666666667, "Malay,English": 0.44666666666666666, "Indonesian,English": 0.54 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.2866666666666667, "Filipino,Vietnamese,Spanish": 0.26666666666666666, "Filipino,Vietnamese,Malay": 0.23333333333333334, "Filipino,Vietnamese,Indonesian": 0.26666666666666666, "Filipino,Vietnamese,English": 0.24, "Filipino,Chinese,Spanish": 0.32666666666666666, "Filipino,Chinese,Malay": 0.28, "Filipino,Chinese,Indonesian": 0.3333333333333333, "Filipino,Chinese,English": 0.3466666666666667, "Filipino,Spanish,Malay": 0.2866666666666667, "Filipino,Spanish,Indonesian": 0.31333333333333335, "Filipino,Spanish,English": 0.3333333333333333, "Filipino,Malay,Indonesian": 0.2733333333333333, "Filipino,Malay,English": 0.22666666666666666, "Filipino,Indonesian,English": 0.28, "Vietnamese,Chinese,Spanish": 0.35333333333333333, "Vietnamese,Chinese,Malay": 0.2733333333333333, "Vietnamese,Chinese,Indonesian": 0.35333333333333333, "Vietnamese,Chinese,English": 0.37333333333333335, "Vietnamese,Spanish,Malay": 0.2866666666666667, "Vietnamese,Spanish,Indonesian": 0.37333333333333335, "Vietnamese,Spanish,English": 0.38666666666666666, "Vietnamese,Malay,Indonesian": 0.29333333333333333, "Vietnamese,Malay,English": 0.28, "Vietnamese,Indonesian,English": 0.3466666666666667, "Chinese,Spanish,Malay": 0.32666666666666666, "Chinese,Spanish,Indonesian": 0.38, "Chinese,Spanish,English": 0.46, "Chinese,Malay,Indonesian": 0.29333333333333333, "Chinese,Malay,English": 0.30666666666666664, "Chinese,Indonesian,English": 0.38, "Spanish,Malay,Indonesian": 0.32666666666666666, "Spanish,Malay,English": 0.3333333333333333, "Spanish,Indonesian,English": 0.42, "Malay,Indonesian,English": 0.3 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.2, "Filipino,Vietnamese,Chinese,Malay": 0.18, "Filipino,Vietnamese,Chinese,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.17333333333333334, "Filipino,Vietnamese,Spanish,Indonesian": 0.21333333333333335, "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, "Filipino,Vietnamese,Malay,Indonesian": 0.18, "Filipino,Vietnamese,Malay,English": 0.14666666666666667, "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay": 0.21333333333333335, "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, "Filipino,Chinese,Spanish,English": 0.26666666666666666, "Filipino,Chinese,Malay,Indonesian": 0.22, "Filipino,Chinese,Malay,English": 0.18666666666666668, "Filipino,Chinese,Indonesian,English": 0.24, "Filipino,Spanish,Malay,Indonesian": 0.22, "Filipino,Spanish,Malay,English": 0.18666666666666668, "Filipino,Spanish,Indonesian,English": 0.24, "Filipino,Malay,Indonesian,English": 0.17333333333333334, "Vietnamese,Chinese,Spanish,Malay": 0.22, "Vietnamese,Chinese,Spanish,Indonesian": 0.26, "Vietnamese,Chinese,Spanish,English": 0.3, "Vietnamese,Chinese,Malay,Indonesian": 0.21333333333333335, "Vietnamese,Chinese,Malay,English": 0.22666666666666666, "Vietnamese,Chinese,Indonesian,English": 0.26666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.24, "Vietnamese,Spanish,Malay,English": 0.23333333333333334, "Vietnamese,Spanish,Indonesian,English": 0.2866666666666667, "Vietnamese,Malay,Indonesian,English": 0.22666666666666666, "Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, "Chinese,Spanish,Malay,English": 0.25333333333333335, "Chinese,Spanish,Indonesian,English": 0.31333333333333335, "Chinese,Malay,Indonesian,English": 0.23333333333333334, "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, "Filipino,Vietnamese,Chinese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16, "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian,English": 0.2, "Filipino,Chinese,Malay,Indonesian,English": 0.16, "Filipino,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.22, "Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, "Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.5136070803246517, "AC3_3": 0.39785714281026796, "AC3_4": 0.31611941369290436, "AC3_5": 0.25620820090148033, "AC3_6": 0.2118856047846738, "AC3_7": 0.17761833579673655 }, "prompt_5": { "overall_acc": 0.5323809523809524, "language_acc": { "Filipino": 0.4, "Vietnamese": 0.49333333333333335, "Chinese": 0.62, "Spanish": 0.56, "Malay": 0.4266666666666667, "Indonesian": 0.54, "English": 0.6866666666666666 }, "consistency_score_2": 0.4990476190476191, "consistency_score_3": 0.3165714285714285, "consistency_score_4": 0.2232380952380953, "consistency_score_5": 0.16666666666666669, "consistency_score_6": 0.13047619047619047, "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { "Filipino,Vietnamese": 0.4066666666666667, "Filipino,Chinese": 0.41333333333333333, "Filipino,Spanish": 0.46, "Filipino,Malay": 0.44, "Filipino,Indonesian": 0.37333333333333335, "Filipino,English": 0.44666666666666666, "Vietnamese,Chinese": 0.49333333333333335, "Vietnamese,Spanish": 0.5266666666666666, "Vietnamese,Malay": 0.48, "Vietnamese,Indonesian": 0.52, "Vietnamese,English": 0.5933333333333334, "Chinese,Spanish": 0.5466666666666666, "Chinese,Malay": 0.44666666666666666, "Chinese,Indonesian": 0.5, "Chinese,English": 0.5933333333333334, "Spanish,Malay": 0.49333333333333335, "Spanish,Indonesian": 0.5466666666666666, "Spanish,English": 0.6133333333333333, "Malay,Indonesian": 0.5533333333333333, "Malay,English": 0.49333333333333335, "Indonesian,English": 0.54 }, "3_combine": { "Filipino,Vietnamese,Chinese": 0.23333333333333334, "Filipino,Vietnamese,Spanish": 0.28, "Filipino,Vietnamese,Malay": 0.24666666666666667, "Filipino,Vietnamese,Indonesian": 0.24, "Filipino,Vietnamese,English": 0.29333333333333333, "Filipino,Chinese,Spanish": 0.2866666666666667, "Filipino,Chinese,Malay": 0.22666666666666666, "Filipino,Chinese,Indonesian": 0.22666666666666666, "Filipino,Chinese,English": 0.3, "Filipino,Spanish,Malay": 0.2733333333333333, "Filipino,Spanish,Indonesian": 0.26, "Filipino,Spanish,English": 0.32666666666666666, "Filipino,Malay,Indonesian": 0.26666666666666666, "Filipino,Malay,English": 0.26, "Filipino,Indonesian,English": 0.26, "Vietnamese,Chinese,Spanish": 0.32666666666666666, "Vietnamese,Chinese,Malay": 0.2866666666666667, "Vietnamese,Chinese,Indonesian": 0.32666666666666666, "Vietnamese,Chinese,English": 0.38666666666666666, "Vietnamese,Spanish,Malay": 0.3466666666666667, "Vietnamese,Spanish,Indonesian": 0.35333333333333333, "Vietnamese,Spanish,English": 0.42, "Vietnamese,Malay,Indonesian": 0.35333333333333333, "Vietnamese,Malay,English": 0.36, "Vietnamese,Indonesian,English": 0.37333333333333335, "Chinese,Spanish,Malay": 0.31333333333333335, "Chinese,Spanish,Indonesian": 0.3466666666666667, "Chinese,Spanish,English": 0.42, "Chinese,Malay,Indonesian": 0.31333333333333335, "Chinese,Malay,English": 0.31333333333333335, "Chinese,Indonesian,English": 0.37333333333333335, "Spanish,Malay,Indonesian": 0.36, "Spanish,Malay,English": 0.36666666666666664, "Spanish,Indonesian,English": 0.4, "Malay,Indonesian,English": 0.36 }, "4_combine": { "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, "Filipino,Vietnamese,Chinese,Malay": 0.14666666666666667, "Filipino,Vietnamese,Chinese,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, "Filipino,Vietnamese,Spanish,Malay": 0.2, "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, "Filipino,Vietnamese,Malay,Indonesian": 0.2, "Filipino,Vietnamese,Malay,English": 0.19333333333333333, "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, "Filipino,Chinese,Spanish,Malay": 0.18666666666666668, "Filipino,Chinese,Spanish,Indonesian": 0.19333333333333333, "Filipino,Chinese,Spanish,English": 0.24666666666666667, "Filipino,Chinese,Malay,Indonesian": 0.16, "Filipino,Chinese,Malay,English": 0.17333333333333334, "Filipino,Chinese,Indonesian,English": 0.19333333333333333, "Filipino,Spanish,Malay,Indonesian": 0.20666666666666667, "Filipino,Spanish,Malay,English": 0.20666666666666667, "Filipino,Spanish,Indonesian,English": 0.20666666666666667, "Filipino,Malay,Indonesian,English": 0.18666666666666668, "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, "Vietnamese,Chinese,Spanish,Indonesian": 0.24, "Vietnamese,Chinese,Spanish,English": 0.2866666666666667, "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, "Vietnamese,Chinese,Malay,English": 0.24, "Vietnamese,Chinese,Indonesian,English": 0.26666666666666666, "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, "Vietnamese,Spanish,Malay,English": 0.2866666666666667, "Vietnamese,Spanish,Indonesian,English": 0.2866666666666667, "Vietnamese,Malay,Indonesian,English": 0.2866666666666667, "Chinese,Spanish,Malay,Indonesian": 0.24, "Chinese,Spanish,Malay,English": 0.26, "Chinese,Spanish,Indonesian,English": 0.29333333333333333, "Chinese,Malay,Indonesian,English": 0.24, "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, "Filipino,Vietnamese,Chinese,Spanish,English": 0.17333333333333334, "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Malay,English": 0.13333333333333333, "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16666666666666666, "Filipino,Vietnamese,Spanish,Malay,English": 0.16, "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Spanish,Malay,Indonesian,English": 0.15333333333333332, "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, "Vietnamese,Chinese,Spanish,Malay,English": 0.20666666666666667, "Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, "Vietnamese,Chinese,Malay,Indonesian,English": 0.2, "Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12, "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12, "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 }, "7_combine": { "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, "AC3_2": 0.5151756583944087, "AC3_3": 0.3970460591217941, "AC3_4": 0.3145704442187555, "AC3_5": 0.25386012712081474, "AC3_6": 0.20958675421030795, "AC3_7": 0.17772478884451898 } }, "cross_logiqa": { "prompt_1": { "overall_acc": 0.4602272727272728, "language_acc": { "Indonesian": 0.45454545454545453, "English": 0.5511363636363636, "Filipino": 0.3465909090909091, "Spanish": 0.45454545454545453, "Chinese": 0.6079545454545454, "Malay": 0.3806818181818182, "Vietnamese": 0.42613636363636365 }, "consistency_score_2": 0.5227272727272727, "consistency_score_3": 0.35211038961038965, "consistency_score_4": 0.2597402597402597, "consistency_score_5": 0.20021645021645015, "consistency_score_6": 0.15746753246753248, "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5454545454545454, "Indonesian,Filipino": 0.42613636363636365, "Indonesian,Spanish": 0.5113636363636364, "Indonesian,Chinese": 0.5568181818181818, "Indonesian,Malay": 0.5227272727272727, "Indonesian,Vietnamese": 0.5681818181818182, "English,Filipino": 0.4375, "English,Spanish": 0.5738636363636364, "English,Chinese": 0.6931818181818182, "English,Malay": 0.45454545454545453, "English,Vietnamese": 0.5852272727272727, "Filipino,Spanish": 0.4772727272727273, "Filipino,Chinese": 0.48295454545454547, "Filipino,Malay": 0.4943181818181818, "Filipino,Vietnamese": 0.5170454545454546, "Spanish,Chinese": 0.5397727272727273, "Spanish,Malay": 0.4943181818181818, "Spanish,Vietnamese": 0.5340909090909091, "Chinese,Malay": 0.4772727272727273, "Chinese,Vietnamese": 0.5795454545454546, "Malay,Vietnamese": 0.5056818181818182 }, "3_combine": { "Indonesian,English,Filipino": 0.3181818181818182, "Indonesian,English,Spanish": 0.3806818181818182, "Indonesian,English,Chinese": 0.4318181818181818, "Indonesian,English,Malay": 0.32954545454545453, "Indonesian,English,Vietnamese": 0.4147727272727273, "Indonesian,Filipino,Spanish": 0.29545454545454547, "Indonesian,Filipino,Chinese": 0.32954545454545453, "Indonesian,Filipino,Malay": 0.2897727272727273, "Indonesian,Filipino,Vietnamese": 0.3465909090909091, "Indonesian,Spanish,Chinese": 0.375, "Indonesian,Spanish,Malay": 0.32954545454545453, "Indonesian,Spanish,Vietnamese": 0.38636363636363635, "Indonesian,Chinese,Malay": 0.3522727272727273, "Indonesian,Chinese,Vietnamese": 0.42045454545454547, "Indonesian,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish": 0.32386363636363635, "English,Filipino,Chinese": 0.36363636363636365, "English,Filipino,Malay": 0.2556818181818182, "English,Filipino,Vietnamese": 0.36363636363636365, "English,Spanish,Chinese": 0.44886363636363635, "English,Spanish,Malay": 0.3352272727272727, "English,Spanish,Vietnamese": 0.4034090909090909, "English,Chinese,Malay": 0.35795454545454547, "English,Chinese,Vietnamese": 0.4772727272727273, "English,Malay,Vietnamese": 0.3181818181818182, "Filipino,Spanish,Chinese": 0.3181818181818182, "Filipino,Spanish,Malay": 0.3068181818181818, "Filipino,Spanish,Vietnamese": 0.3352272727272727, "Filipino,Chinese,Malay": 0.2897727272727273, "Filipino,Chinese,Vietnamese": 0.375, "Filipino,Malay,Vietnamese": 0.3125, "Spanish,Chinese,Malay": 0.32386363636363635, "Spanish,Chinese,Vietnamese": 0.3977272727272727, "Spanish,Malay,Vietnamese": 0.3352272727272727, "Chinese,Malay,Vietnamese": 0.32954545454545453 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.23863636363636365, "Indonesian,English,Filipino,Chinese": 0.2727272727272727, "Indonesian,English,Filipino,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Vietnamese": 0.2840909090909091, "Indonesian,English,Spanish,Chinese": 0.3125, "Indonesian,English,Spanish,Malay": 0.25, "Indonesian,English,Spanish,Vietnamese": 0.3181818181818182, "Indonesian,English,Chinese,Malay": 0.2727272727272727, "Indonesian,English,Chinese,Vietnamese": 0.3409090909090909, "Indonesian,English,Malay,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Spanish,Chinese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Malay": 0.20454545454545456, "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, "Indonesian,Filipino,Chinese,Malay": 0.2159090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.32954545454545453, "Indonesian,Spanish,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, "English,Filipino,Spanish,Chinese": 0.2840909090909091, "English,Filipino,Spanish,Malay": 0.21022727272727273, "English,Filipino,Spanish,Vietnamese": 0.2784090909090909, "English,Filipino,Chinese,Malay": 0.20454545454545456, "English,Filipino,Chinese,Vietnamese": 0.3181818181818182, "English,Filipino,Malay,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese,Malay": 0.2727272727272727, "English,Spanish,Chinese,Vietnamese": 0.3352272727272727, "English,Spanish,Malay,Vietnamese": 0.26136363636363635, "English,Chinese,Malay,Vietnamese": 0.25, "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, "Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.2159090909090909, "Indonesian,English,Filipino,Spanish,Malay": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2215909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Chinese,Malay": 0.2159090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.21022727272727273, "Indonesian,English,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, "English,Filipino,Spanish,Chinese,Vietnamese": 0.25, "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, "English,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, "AC3_2": 0.4894902784578217, "AC3_3": 0.3989740025669747, "AC3_4": 0.332069283545145, "AC3_5": 0.27903988674036473, "AC3_6": 0.23464938474280295, "AC3_7": 0.1966019417139787 }, "prompt_2": { "overall_acc": 0.46185064935064934, "language_acc": { "Indonesian": 0.45454545454545453, "English": 0.5681818181818182, "Filipino": 0.3352272727272727, "Spanish": 0.4659090909090909, "Chinese": 0.5738636363636364, "Malay": 0.42613636363636365, "Vietnamese": 0.4090909090909091 }, "consistency_score_2": 0.5094696969696969, "consistency_score_3": 0.33068181818181813, "consistency_score_4": 0.24090909090909085, "consistency_score_5": 0.18695887445887452, "consistency_score_6": 0.15097402597402598, "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5227272727272727, "Indonesian,Filipino": 0.4715909090909091, "Indonesian,Spanish": 0.5397727272727273, "Indonesian,Chinese": 0.5625, "Indonesian,Malay": 0.5170454545454546, "Indonesian,Vietnamese": 0.5511363636363636, "English,Filipino": 0.44886363636363635, "English,Spanish": 0.6022727272727273, "English,Chinese": 0.5965909090909091, "English,Malay": 0.4318181818181818, "English,Vietnamese": 0.4943181818181818, "Filipino,Spanish": 0.4772727272727273, "Filipino,Chinese": 0.4659090909090909, "Filipino,Malay": 0.4659090909090909, "Filipino,Vietnamese": 0.4715909090909091, "Spanish,Chinese": 0.5340909090909091, "Spanish,Malay": 0.5170454545454546, "Spanish,Vietnamese": 0.5625, "Chinese,Malay": 0.4943181818181818, "Chinese,Vietnamese": 0.4943181818181818, "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { "Indonesian,English,Filipino": 0.3068181818181818, "Indonesian,English,Spanish": 0.38636363636363635, "Indonesian,English,Chinese": 0.3806818181818182, "Indonesian,English,Malay": 0.3068181818181818, "Indonesian,English,Vietnamese": 0.3465909090909091, "Indonesian,Filipino,Spanish": 0.32954545454545453, "Indonesian,Filipino,Chinese": 0.3068181818181818, "Indonesian,Filipino,Malay": 0.29545454545454547, "Indonesian,Filipino,Vietnamese": 0.32386363636363635, "Indonesian,Spanish,Chinese": 0.3693181818181818, "Indonesian,Spanish,Malay": 0.3352272727272727, "Indonesian,Spanish,Vietnamese": 0.3806818181818182, "Indonesian,Chinese,Malay": 0.3465909090909091, "Indonesian,Chinese,Vietnamese": 0.36363636363636365, "Indonesian,Malay,Vietnamese": 0.3409090909090909, "English,Filipino,Spanish": 0.3465909090909091, "English,Filipino,Chinese": 0.32954545454545453, "English,Filipino,Malay": 0.23863636363636365, "English,Filipino,Vietnamese": 0.30113636363636365, "English,Spanish,Chinese": 0.42613636363636365, "English,Spanish,Malay": 0.3409090909090909, "English,Spanish,Vietnamese": 0.4034090909090909, "English,Chinese,Malay": 0.32954545454545453, "English,Chinese,Vietnamese": 0.36363636363636365, "English,Malay,Vietnamese": 0.2727272727272727, "Filipino,Spanish,Chinese": 0.3068181818181818, "Filipino,Spanish,Malay": 0.30113636363636365, "Filipino,Spanish,Vietnamese": 0.3181818181818182, "Filipino,Chinese,Malay": 0.2727272727272727, "Filipino,Chinese,Vietnamese": 0.30113636363636365, "Filipino,Malay,Vietnamese": 0.2784090909090909, "Spanish,Chinese,Malay": 0.32954545454545453, "Spanish,Chinese,Vietnamese": 0.35795454545454547, "Spanish,Malay,Vietnamese": 0.32386363636363635, "Chinese,Malay,Vietnamese": 0.3125 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.26136363636363635, "Indonesian,English,Filipino,Chinese": 0.23295454545454544, "Indonesian,English,Filipino,Malay": 0.19318181818181818, "Indonesian,English,Filipino,Vietnamese": 0.23295454545454544, "Indonesian,English,Spanish,Chinese": 0.30113636363636365, "Indonesian,English,Spanish,Malay": 0.25, "Indonesian,English,Spanish,Vietnamese": 0.30113636363636365, "Indonesian,English,Chinese,Malay": 0.26136363636363635, "Indonesian,English,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,English,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese": 0.22727272727272727, "Indonesian,Filipino,Spanish,Malay": 0.22727272727272727, "Indonesian,Filipino,Spanish,Vietnamese": 0.25, "Indonesian,Filipino,Chinese,Malay": 0.20454545454545456, "Indonesian,Filipino,Chinese,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Chinese,Malay": 0.26136363636363635, "Indonesian,Spanish,Chinese,Vietnamese": 0.2840909090909091, "Indonesian,Spanish,Malay,Vietnamese": 0.26136363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.2556818181818182, "English,Filipino,Spanish,Chinese": 0.2727272727272727, "English,Filipino,Spanish,Malay": 0.2159090909090909, "English,Filipino,Spanish,Vietnamese": 0.26704545454545453, "English,Filipino,Chinese,Malay": 0.19318181818181818, "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, "English,Filipino,Malay,Vietnamese": 0.17613636363636365, "English,Spanish,Chinese,Malay": 0.26704545454545453, "English,Spanish,Chinese,Vietnamese": 0.30113636363636365, "English,Spanish,Malay,Vietnamese": 0.23295454545454544, "English,Chinese,Malay,Vietnamese": 0.22727272727272727, "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, "Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "Indonesian,English,Filipino,Chinese,Malay": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Malay": 0.2159090909090909, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.24431818181818182, "Indonesian,English,Spanish,Malay,Vietnamese": 0.20454545454545456, "Indonesian,English,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "English,Filipino,Spanish,Chinese,Vietnamese": 0.2159090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.1875, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, "AC3_2": 0.4844929095474865, "AC3_3": 0.38541162335459483, "AC3_4": 0.3166488166037587, "AC3_5": 0.26617080788332814, "AC3_6": 0.22756084970914708, "AC3_7": 0.19674965418500753 }, "prompt_3": { "overall_acc": 0.40909090909090917, "language_acc": { "Indonesian": 0.3693181818181818, "English": 0.5, "Filipino": 0.2784090909090909, "Spanish": 0.44886363636363635, "Chinese": 0.5511363636363636, "Malay": 0.3352272727272727, "Vietnamese": 0.3806818181818182 }, "consistency_score_2": 0.4334415584415584, "consistency_score_3": 0.2407467532467533, "consistency_score_4": 0.1530844155844156, "consistency_score_5": 0.10660173160173161, "consistency_score_6": 0.07954545454545454, "consistency_score_7": 0.0625, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5056818181818182, "Indonesian,Filipino": 0.3977272727272727, "Indonesian,Spanish": 0.4659090909090909, "Indonesian,Chinese": 0.4602272727272727, "Indonesian,Malay": 0.4090909090909091, "Indonesian,Vietnamese": 0.44886363636363635, "English,Filipino": 0.32386363636363635, "English,Spanish": 0.48863636363636365, "English,Chinese": 0.4943181818181818, "English,Malay": 0.36363636363636365, "English,Vietnamese": 0.4602272727272727, "Filipino,Spanish": 0.3522727272727273, "Filipino,Chinese": 0.3977272727272727, "Filipino,Malay": 0.4772727272727273, "Filipino,Vietnamese": 0.39204545454545453, "Spanish,Chinese": 0.4943181818181818, "Spanish,Malay": 0.42613636363636365, "Spanish,Vietnamese": 0.4602272727272727, "Chinese,Malay": 0.3977272727272727, "Chinese,Vietnamese": 0.45454545454545453, "Malay,Vietnamese": 0.4318181818181818 }, "3_combine": { "Indonesian,English,Filipino": 0.2159090909090909, "Indonesian,English,Spanish": 0.3068181818181818, "Indonesian,English,Chinese": 0.3068181818181818, "Indonesian,English,Malay": 0.23295454545454544, "Indonesian,English,Vietnamese": 0.2897727272727273, "Indonesian,Filipino,Spanish": 0.19886363636363635, "Indonesian,Filipino,Chinese": 0.23863636363636365, "Indonesian,Filipino,Malay": 0.24431818181818182, "Indonesian,Filipino,Vietnamese": 0.22727272727272727, "Indonesian,Spanish,Chinese": 0.29545454545454547, "Indonesian,Spanish,Malay": 0.23295454545454544, "Indonesian,Spanish,Vietnamese": 0.2784090909090909, "Indonesian,Chinese,Malay": 0.23295454545454544, "Indonesian,Chinese,Vietnamese": 0.25, "Indonesian,Malay,Vietnamese": 0.2215909090909091, "English,Filipino,Spanish": 0.17613636363636365, "English,Filipino,Chinese": 0.19886363636363635, "English,Filipino,Malay": 0.1875, "English,Filipino,Vietnamese": 0.19886363636363635, "English,Spanish,Chinese": 0.3465909090909091, "English,Spanish,Malay": 0.2159090909090909, "English,Spanish,Vietnamese": 0.2897727272727273, "English,Chinese,Malay": 0.21022727272727273, "English,Chinese,Vietnamese": 0.29545454545454547, "English,Malay,Vietnamese": 0.21022727272727273, "Filipino,Spanish,Chinese": 0.23295454545454544, "Filipino,Spanish,Malay": 0.21022727272727273, "Filipino,Spanish,Vietnamese": 0.20454545454545456, "Filipino,Chinese,Malay": 0.23295454545454544, "Filipino,Chinese,Vietnamese": 0.23295454545454544, "Filipino,Malay,Vietnamese": 0.2556818181818182, "Spanish,Chinese,Malay": 0.24431818181818182, "Spanish,Chinese,Vietnamese": 0.2784090909090909, "Spanish,Malay,Vietnamese": 0.2215909090909091, "Chinese,Malay,Vietnamese": 0.21022727272727273 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.13068181818181818, "Indonesian,English,Filipino,Chinese": 0.1590909090909091, "Indonesian,English,Filipino,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese": 0.23295454545454544, "Indonesian,English,Spanish,Malay": 0.14772727272727273, "Indonesian,English,Spanish,Vietnamese": 0.20454545454545456, "Indonesian,English,Chinese,Malay": 0.1590909090909091, "Indonesian,English,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,English,Malay,Vietnamese": 0.14204545454545456, "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, "Indonesian,Filipino,Spanish,Malay": 0.13068181818181818, "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, "Indonesian,Filipino,Chinese,Malay": 0.1534090909090909, "Indonesian,Filipino,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,Spanish,Malay,Vietnamese": 0.13636363636363635, "Indonesian,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese": 0.16477272727272727, "English,Filipino,Spanish,Malay": 0.11363636363636363, "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, "English,Filipino,Chinese,Malay": 0.125, "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, "English,Filipino,Malay,Vietnamese": 0.13068181818181818, "English,Spanish,Chinese,Malay": 0.16477272727272727, "English,Spanish,Chinese,Vietnamese": 0.22727272727272727, "English,Spanish,Malay,Vietnamese": 0.13636363636363635, "English,Chinese,Malay,Vietnamese": 0.13068181818181818, "Filipino,Spanish,Chinese,Malay": 0.13636363636363635, "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, "Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "Spanish,Chinese,Malay,Vietnamese": 0.125 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.125, "Indonesian,English,Filipino,Spanish,Malay": 0.09090909090909091, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,English,Filipino,Malay,Vietnamese": 0.10227272727272728, "Indonesian,English,Spanish,Chinese,Malay": 0.125, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, "Indonesian,English,Chinese,Malay,Vietnamese": 0.09659090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, "English,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, "English,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 } }, "AC3_2": 0.4209143457198789, "AC3_3": 0.30311357386383175, "AC3_4": 0.22279683927467855, "AC3_5": 0.16913097392503967, "AC3_6": 0.1331923889790845, "AC3_7": 0.10843373491676586 }, "prompt_4": { "overall_acc": 0.4642857142857143, "language_acc": { "Indonesian": 0.4659090909090909, "English": 0.5397727272727273, "Filipino": 0.36363636363636365, "Spanish": 0.4431818181818182, "Chinese": 0.5852272727272727, "Malay": 0.4147727272727273, "Vietnamese": 0.4375 }, "consistency_score_2": 0.507034632034632, "consistency_score_3": 0.33003246753246757, "consistency_score_4": 0.24237012987012982, "consistency_score_5": 0.1915584415584416, "consistency_score_6": 0.1590909090909091, "consistency_score_7": 0.13636363636363635, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5625, "Indonesian,Filipino": 0.42045454545454547, "Indonesian,Spanish": 0.5284090909090909, "Indonesian,Chinese": 0.5454545454545454, "Indonesian,Malay": 0.5625, "Indonesian,Vietnamese": 0.5113636363636364, "English,Filipino": 0.45454545454545453, "English,Spanish": 0.5795454545454546, "English,Chinese": 0.6193181818181818, "English,Malay": 0.4772727272727273, "English,Vietnamese": 0.5454545454545454, "Filipino,Spanish": 0.4034090909090909, "Filipino,Chinese": 0.45454545454545453, "Filipino,Malay": 0.48295454545454547, "Filipino,Vietnamese": 0.4375, "Spanish,Chinese": 0.5056818181818182, "Spanish,Malay": 0.4943181818181818, "Spanish,Vietnamese": 0.48863636363636365, "Chinese,Malay": 0.5113636363636364, "Chinese,Vietnamese": 0.5454545454545454, "Malay,Vietnamese": 0.5170454545454546 }, "3_combine": { "Indonesian,English,Filipino": 0.29545454545454547, "Indonesian,English,Spanish": 0.3977272727272727, "Indonesian,English,Chinese": 0.42045454545454547, "Indonesian,English,Malay": 0.36363636363636365, "Indonesian,English,Vietnamese": 0.375, "Indonesian,Filipino,Spanish": 0.2727272727272727, "Indonesian,Filipino,Chinese": 0.30113636363636365, "Indonesian,Filipino,Malay": 0.32386363636363635, "Indonesian,Filipino,Vietnamese": 0.26136363636363635, "Indonesian,Spanish,Chinese": 0.35795454545454547, "Indonesian,Spanish,Malay": 0.3522727272727273, "Indonesian,Spanish,Vietnamese": 0.3125, "Indonesian,Chinese,Malay": 0.375, "Indonesian,Chinese,Vietnamese": 0.375, "Indonesian,Malay,Vietnamese": 0.3522727272727273, "English,Filipino,Spanish": 0.30113636363636365, "English,Filipino,Chinese": 0.32954545454545453, "English,Filipino,Malay": 0.29545454545454547, "English,Filipino,Vietnamese": 0.2897727272727273, "English,Spanish,Chinese": 0.4034090909090909, "English,Spanish,Malay": 0.32954545454545453, "English,Spanish,Vietnamese": 0.3465909090909091, "English,Chinese,Malay": 0.3522727272727273, "English,Chinese,Vietnamese": 0.4147727272727273, "English,Malay,Vietnamese": 0.3352272727272727, "Filipino,Spanish,Chinese": 0.2727272727272727, "Filipino,Spanish,Malay": 0.2727272727272727, "Filipino,Spanish,Vietnamese": 0.24431818181818182, "Filipino,Chinese,Malay": 0.3068181818181818, "Filipino,Chinese,Vietnamese": 0.2897727272727273, "Filipino,Malay,Vietnamese": 0.30113636363636365, "Spanish,Chinese,Malay": 0.3352272727272727, "Spanish,Chinese,Vietnamese": 0.32386363636363635, "Spanish,Malay,Vietnamese": 0.3125, "Chinese,Malay,Vietnamese": 0.35795454545454547 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.22727272727272727, "Indonesian,English,Filipino,Chinese": 0.25, "Indonesian,English,Filipino,Malay": 0.22727272727272727, "Indonesian,English,Filipino,Vietnamese": 0.21022727272727273, "Indonesian,English,Spanish,Chinese": 0.3181818181818182, "Indonesian,English,Spanish,Malay": 0.2840909090909091, "Indonesian,English,Spanish,Vietnamese": 0.26704545454545453, "Indonesian,English,Chinese,Malay": 0.29545454545454547, "Indonesian,English,Chinese,Vietnamese": 0.3181818181818182, "Indonesian,English,Malay,Vietnamese": 0.2840909090909091, "Indonesian,Filipino,Spanish,Chinese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Malay": 0.21022727272727273, "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Chinese,Malay": 0.24431818181818182, "Indonesian,Filipino,Chinese,Vietnamese": 0.2215909090909091, "Indonesian,Filipino,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Spanish,Chinese,Malay": 0.26704545454545453, "Indonesian,Spanish,Chinese,Vietnamese": 0.26136363636363635, "Indonesian,Spanish,Malay,Vietnamese": 0.23863636363636365, "Indonesian,Chinese,Malay,Vietnamese": 0.2784090909090909, "English,Filipino,Spanish,Chinese": 0.24431818181818182, "English,Filipino,Spanish,Malay": 0.21022727272727273, "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, "English,Filipino,Chinese,Malay": 0.23295454545454544, "English,Filipino,Chinese,Vietnamese": 0.24431818181818182, "English,Filipino,Malay,Vietnamese": 0.21022727272727273, "English,Spanish,Chinese,Malay": 0.2727272727272727, "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, "English,Spanish,Malay,Vietnamese": 0.25, "English,Chinese,Malay,Vietnamese": 0.2727272727272727, "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, "Filipino,Spanish,Chinese,Vietnamese": 0.1875, "Filipino,Spanish,Malay,Vietnamese": 0.1875, "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, "Spanish,Chinese,Malay,Vietnamese": 0.25 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, "Indonesian,English,Filipino,Spanish,Malay": 0.18181818181818182, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.16477272727272727, "Indonesian,English,Filipino,Chinese,Malay": 0.19886363636363635, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.19318181818181818, "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, "Indonesian,English,Spanish,Chinese,Malay": 0.24431818181818182, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.23863636363636365, "Indonesian,English,Spanish,Malay,Vietnamese": 0.22727272727272727, "Indonesian,English,Chinese,Malay,Vietnamese": 0.24431818181818182, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, "English,Filipino,Spanish,Chinese,Malay": 0.1875, "English,Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, "English,Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 } }, "AC3_2": 0.48471945876426314, "AC3_3": 0.38581355284039504, "AC3_4": 0.31848314786605947, "AC3_5": 0.2712164073136675, "AC3_6": 0.23697916662865126, "AC3_7": 0.21081081077571362 }, "prompt_5": { "overall_acc": 0.45860389610389607, "language_acc": { "Indonesian": 0.4659090909090909, "English": 0.5568181818181818, "Filipino": 0.35795454545454547, "Spanish": 0.4318181818181818, "Chinese": 0.5909090909090909, "Malay": 0.3693181818181818, "Vietnamese": 0.4375 }, "consistency_score_2": 0.4802489177489177, "consistency_score_3": 0.29740259740259756, "consistency_score_4": 0.21087662337662336, "consistency_score_5": 0.16152597402597405, "consistency_score_6": 0.12987012987012989, "consistency_score_7": 0.10795454545454546, "detailed_consistency_score": { "2_combine": { "Indonesian,English": 0.5284090909090909, "Indonesian,Filipino": 0.4090909090909091, "Indonesian,Spanish": 0.5, "Indonesian,Chinese": 0.5681818181818182, "Indonesian,Malay": 0.5227272727272727, "Indonesian,Vietnamese": 0.5284090909090909, "English,Filipino": 0.3693181818181818, "English,Spanish": 0.5454545454545454, "English,Chinese": 0.5909090909090909, "English,Malay": 0.48863636363636365, "English,Vietnamese": 0.48295454545454547, "Filipino,Spanish": 0.4375, "Filipino,Chinese": 0.4034090909090909, "Filipino,Malay": 0.42613636363636365, "Filipino,Vietnamese": 0.4318181818181818, "Spanish,Chinese": 0.4943181818181818, "Spanish,Malay": 0.4715909090909091, "Spanish,Vietnamese": 0.5, "Chinese,Malay": 0.4431818181818182, "Chinese,Vietnamese": 0.48863636363636365, "Malay,Vietnamese": 0.45454545454545453 }, "3_combine": { "Indonesian,English,Filipino": 0.23863636363636365, "Indonesian,English,Spanish": 0.35795454545454547, "Indonesian,English,Chinese": 0.38636363636363635, "Indonesian,English,Malay": 0.32386363636363635, "Indonesian,English,Vietnamese": 0.3352272727272727, "Indonesian,Filipino,Spanish": 0.2784090909090909, "Indonesian,Filipino,Chinese": 0.2727272727272727, "Indonesian,Filipino,Malay": 0.26704545454545453, "Indonesian,Filipino,Vietnamese": 0.2840909090909091, "Indonesian,Spanish,Chinese": 0.3465909090909091, "Indonesian,Spanish,Malay": 0.3068181818181818, "Indonesian,Spanish,Vietnamese": 0.32954545454545453, "Indonesian,Chinese,Malay": 0.32386363636363635, "Indonesian,Chinese,Vietnamese": 0.35795454545454547, "Indonesian,Malay,Vietnamese": 0.3068181818181818, "English,Filipino,Spanish": 0.2556818181818182, "English,Filipino,Chinese": 0.26704545454545453, "English,Filipino,Malay": 0.24431818181818182, "English,Filipino,Vietnamese": 0.23863636363636365, "English,Spanish,Chinese": 0.375, "English,Spanish,Malay": 0.32386363636363635, "English,Spanish,Vietnamese": 0.3465909090909091, "English,Chinese,Malay": 0.32386363636363635, "English,Chinese,Vietnamese": 0.3352272727272727, "English,Malay,Vietnamese": 0.2897727272727273, "Filipino,Spanish,Chinese": 0.26136363636363635, "Filipino,Spanish,Malay": 0.25, "Filipino,Spanish,Vietnamese": 0.2727272727272727, "Filipino,Chinese,Malay": 0.23295454545454544, "Filipino,Chinese,Vietnamese": 0.2784090909090909, "Filipino,Malay,Vietnamese": 0.25, "Spanish,Chinese,Malay": 0.2784090909090909, "Spanish,Chinese,Vietnamese": 0.3181818181818182, "Spanish,Malay,Vietnamese": 0.2784090909090909, "Chinese,Malay,Vietnamese": 0.2727272727272727 }, "4_combine": { "Indonesian,English,Filipino,Spanish": 0.1875, "Indonesian,English,Filipino,Chinese": 0.19886363636363635, "Indonesian,English,Filipino,Malay": 0.17613636363636365, "Indonesian,English,Filipino,Vietnamese": 0.19318181818181818, "Indonesian,English,Spanish,Chinese": 0.2727272727272727, "Indonesian,English,Spanish,Malay": 0.23295454545454544, "Indonesian,English,Spanish,Vietnamese": 0.26136363636363635, "Indonesian,English,Chinese,Malay": 0.25, "Indonesian,English,Chinese,Vietnamese": 0.26136363636363635, "Indonesian,English,Malay,Vietnamese": 0.23295454545454544, "Indonesian,Filipino,Spanish,Chinese": 0.21022727272727273, "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, "Indonesian,Filipino,Spanish,Vietnamese": 0.21022727272727273, "Indonesian,Filipino,Chinese,Malay": 0.18181818181818182, "Indonesian,Filipino,Chinese,Vietnamese": 0.22727272727272727, "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, "Indonesian,Spanish,Chinese,Malay": 0.22727272727272727, "Indonesian,Spanish,Chinese,Vietnamese": 0.25, "Indonesian,Spanish,Malay,Vietnamese": 0.2215909090909091, "Indonesian,Chinese,Malay,Vietnamese": 0.22727272727272727, "English,Filipino,Spanish,Chinese": 0.20454545454545456, "English,Filipino,Spanish,Malay": 0.18181818181818182, "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, "English,Filipino,Chinese,Malay": 0.18181818181818182, "English,Filipino,Chinese,Vietnamese": 0.19886363636363635, "English,Filipino,Malay,Vietnamese": 0.17045454545454544, "English,Spanish,Chinese,Malay": 0.23863636363636365, "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, "English,Spanish,Malay,Vietnamese": 0.22727272727272727, "English,Chinese,Malay,Vietnamese": 0.2159090909090909, "Filipino,Spanish,Chinese,Malay": 0.17045454545454544, "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, "Filipino,Chinese,Malay,Vietnamese": 0.1875, "Spanish,Chinese,Malay,Vietnamese": 0.1875 }, "5_combine": { "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, "Indonesian,English,Filipino,Chinese,Malay": 0.14772727272727273, "Indonesian,English,Filipino,Chinese,Vietnamese": 0.16477272727272727, "Indonesian,English,Filipino,Malay,Vietnamese": 0.14772727272727273, "Indonesian,English,Spanish,Chinese,Malay": 0.19318181818181818, "Indonesian,English,Spanish,Chinese,Vietnamese": 0.19886363636363635, "Indonesian,English,Spanish,Malay,Vietnamese": 0.19318181818181818, "Indonesian,English,Chinese,Malay,Vietnamese": 0.1875, "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, "English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, "English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, "English,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.125, "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.125, "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "7_combine": { "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 } }, "AC3_2": 0.4691768965404678, "AC3_3": 0.3608169798373228, "AC3_4": 0.2889071100630347, "AC3_5": 0.23890621808889714, "AC3_6": 0.20241827134939147, "AC3_7": 0.17476881997436225 } }, "sg_eval": { "prompt_1": { "accuracy": 0.6116504854368932 }, "prompt_2": { "accuracy": 0.5922330097087378 }, "prompt_3": { "accuracy": 0.5728155339805825 }, "prompt_4": { "accuracy": 0.6213592233009708 }, "prompt_5": { "accuracy": 0.6213592233009708 } }, "cn_eval": { "prompt_1": { "accuracy": 0.6 }, "prompt_2": { "accuracy": 0.638095238095238 }, "prompt_3": { "accuracy": 0.6761904761904762 }, "prompt_4": { "accuracy": 0.6476190476190476 }, "prompt_5": { "accuracy": 0.6095238095238096 } }, "us_eval": { "prompt_1": { "accuracy": 0.6728971962616822 }, "prompt_2": { "accuracy": 0.6355140186915887 }, "prompt_3": { "accuracy": 0.6728971962616822 }, "prompt_4": { "accuracy": 0.6635514018691588 }, "prompt_5": { "accuracy": 0.6355140186915887 } }, "ph_eval": { "prompt_1": { "accuracy": 0.42, "category_acc": { "brand": 0.4, "demographics": 0.6, "biology": 0.4, "history": 0.26666666666666666, "literature": 0.3, "politics": 0.5, "culture": 0.3, "film": 0.4, "law": 0.4, "geography": 0.8 } }, "prompt_2": { "accuracy": 0.37, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.2, "literature": 0.2, "politics": 0.5, "culture": 0.4, "film": 0.4, "law": 0.5, "geography": 0.6 } }, "prompt_3": { "accuracy": 0.38, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.4, "history": 0.2, "literature": 0.3, "politics": 0.5, "culture": 0.4, "film": 0.3, "law": 0.7, "geography": 0.6 } }, "prompt_4": { "accuracy": 0.4, "category_acc": { "brand": 0.3, "demographics": 0.4, "biology": 0.4, "history": 0.2, "literature": 0.3, "politics": 0.6, "culture": 0.5, "film": 0.3, "law": 0.6, "geography": 0.5 } }, "prompt_5": { "accuracy": 0.4, "category_acc": { "brand": 0.4, "demographics": 0.4, "biology": 0.4, "history": 0.2, "literature": 0.3, "politics": 0.5, "culture": 0.5, "film": 0.3, "law": 0.5, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { "bleu_score": 0.18328104784835536 }, "prompt_2": { "bleu_score": 0.15895050867572064 }, "prompt_3": { "bleu_score": 0.17203662866668645 }, "prompt_4": { "bleu_score": 0.199134691773029 }, "prompt_5": { "bleu_score": 0.13039131852856445 } }, "indommlu": { "prompt_1": { "accuracy": 0.4498297616663329, "category_acc": { "History": 0.3855421686746988, "Geography": 0.4142857142857143, "Lampungic": 0.32653061224489793, "Social science": 0.654424040066778, "Balinese": 0.28450106157112526, "Makassarese": 0.3279569892473118, "Banjarese": 0.3055555555555556, "Chemistry": 0.3094890510948905, "Biology": 0.40236686390532544, "Science": 0.5675954592363261, "Christian religion": 0.5621890547263682, "Art": 0.5457570715474209, "Islam religion": 0.5135135135135135, "Hindu religion": 0.36666666666666664, "Madurese": 0.3152542372881356, "Sport": 0.5540540540540541, "Indonesian language": 0.524906600249066, "Physics": 0.3898989898989899, "Minangkabau culture": 0.3869346733668342, "Dayak language": 0.27522935779816515, "Sociology": 0.4879032258064516, "Economy": 0.45286885245901637, "Sundanese": 0.34917891097666376, "Javanese": 0.31350806451612906, "Civic education": 0.5236051502145923 } }, "prompt_2": { "accuracy": 0.4529007276854263, "category_acc": { "History": 0.38755020080321284, "Geography": 0.4326530612244898, "Lampungic": 0.3333333333333333, "Social science": 0.662771285475793, "Balinese": 0.26963906581740976, "Makassarese": 0.3118279569892473, "Banjarese": 0.2847222222222222, "Chemistry": 0.3124087591240876, "Biology": 0.41893491124260357, "Science": 0.5717234262125903, "Christian religion": 0.5074626865671642, "Art": 0.5474209650582362, "Islam religion": 0.5064011379800853, "Hindu religion": 0.3466666666666667, "Madurese": 0.30847457627118646, "Sport": 0.527027027027027, "Indonesian language": 0.5345579078455791, "Physics": 0.39595959595959596, "Minangkabau culture": 0.3869346733668342, "Dayak language": 0.3302752293577982, "Sociology": 0.4778225806451613, "Economy": 0.4262295081967213, "Sundanese": 0.35350043215211757, "Javanese": 0.32661290322580644, "Civic education": 0.5336194563662375 } }, "prompt_3": { "accuracy": 0.45029708258228185, "category_acc": { "History": 0.39156626506024095, "Geography": 0.42448979591836733, "Lampungic": 0.30612244897959184, "Social science": 0.662771285475793, "Balinese": 0.2484076433121019, "Makassarese": 0.3225806451612903, "Banjarese": 0.2847222222222222, "Chemistry": 0.30802919708029197, "Biology": 0.40946745562130177, "Science": 0.5603715170278638, "Christian religion": 0.5373134328358209, "Art": 0.5440931780366056, "Islam religion": 0.5220483641536273, "Hindu religion": 0.3933333333333333, "Madurese": 0.2983050847457627, "Sport": 0.5202702702702703, "Indonesian language": 0.5292652552926526, "Physics": 0.4121212121212121, "Minangkabau culture": 0.34673366834170855, "Dayak language": 0.3302752293577982, "Sociology": 0.4879032258064516, "Economy": 0.430327868852459, "Sundanese": 0.3560933448573898, "Javanese": 0.3235887096774194, "Civic education": 0.5178826895565093 } }, "prompt_4": { "accuracy": 0.4494959610120836, "category_acc": { "History": 0.37349397590361444, "Geography": 0.42653061224489797, "Lampungic": 0.3197278911564626, "Social science": 0.662771285475793, "Balinese": 0.2632696390658174, "Makassarese": 0.3333333333333333, "Banjarese": 0.3333333333333333, "Chemistry": 0.3167883211678832, "Biology": 0.40946745562130177, "Science": 0.564499484004128, "Christian religion": 0.5373134328358209, "Art": 0.5424292845257903, "Islam religion": 0.5291607396870555, "Hindu religion": 0.37333333333333335, "Madurese": 0.29152542372881357, "Sport": 0.5067567567567568, "Indonesian language": 0.5252179327521793, "Physics": 0.40606060606060607, "Minangkabau culture": 0.3417085427135678, "Dayak language": 0.3486238532110092, "Sociology": 0.4838709677419355, "Economy": 0.44672131147540983, "Sundanese": 0.34485738980121, "Javanese": 0.3125, "Civic education": 0.5236051502145923 } }, "prompt_5": { "accuracy": 0.4462914747312905, "category_acc": { "History": 0.38755020080321284, "Geography": 0.4142857142857143, "Lampungic": 0.3197278911564626, "Social science": 0.664440734557596, "Balinese": 0.2760084925690021, "Makassarese": 0.3655913978494624, "Banjarese": 0.3125, "Chemistry": 0.291970802919708, "Biology": 0.3905325443786982, "Science": 0.5624355005159959, "Christian religion": 0.5422885572139303, "Art": 0.5524126455906821, "Islam religion": 0.519203413940256, "Hindu religion": 0.3466666666666667, "Madurese": 0.3389830508474576, "Sport": 0.5, "Indonesian language": 0.5199252801992528, "Physics": 0.3838383838383838, "Minangkabau culture": 0.32663316582914576, "Dayak language": 0.3119266055045872, "Sociology": 0.4838709677419355, "Economy": 0.4426229508196721, "Sundanese": 0.35177182368193605, "Javanese": 0.3215725806451613, "Civic education": 0.5050071530758226 } } }, "flores_ind2eng": { "prompt_1": { "bleu_score": 0.3044304147296206 }, "prompt_2": { "bleu_score": 0.29821649291017144 }, "prompt_3": { "bleu_score": 0.2928519299374442 }, "prompt_4": { "bleu_score": 0.3136684772191943 }, "prompt_5": { "bleu_score": 0.28063486126359444 } }, "flores_vie2eng": { "prompt_1": { "bleu_score": 0.2516919772808757 }, "prompt_2": { "bleu_score": 0.23887910636911036 }, "prompt_3": { "bleu_score": 0.23924571820883456 }, "prompt_4": { "bleu_score": 0.2584865607862379 }, "prompt_5": { "bleu_score": 0.23683830918379803 } }, "flores_zho2eng": { "prompt_1": { "bleu_score": 0.15584366135182787 }, "prompt_2": { "bleu_score": 0.1440599297310671 }, "prompt_3": { "bleu_score": 0.15144528864585652 }, "prompt_4": { "bleu_score": 0.17181780373780153 }, "prompt_5": { "bleu_score": 0.20064653927085144 } }, "flores_zsm2eng": { "prompt_1": { "bleu_score": 0.28164835459459403 }, "prompt_2": { "bleu_score": 0.2738665326663029 }, "prompt_3": { "bleu_score": 0.2663916384980528 }, "prompt_4": { "bleu_score": 0.2854299944610882 }, "prompt_5": { "bleu_score": 0.25194925478011515 } }, "mmlu": { "prompt_1": { "accuracy": 0.5717619603267211 }, "prompt_2": { "accuracy": 0.5577596266044341 }, "prompt_3": { "accuracy": 0.588098016336056 }, "prompt_4": { "accuracy": 0.5565927654609102 }, "prompt_5": { "accuracy": 0.5612602100350058 } }, "mmlu_full": { "prompt_1": { "accuracy": 0.562531283518055, "category_acc": { "high_school_european_history": 0.7621951219512195, "business_ethics": 0.6262626262626263, "clinical_knowledge": 0.6325757575757576, "medical_genetics": 0.6161616161616161, "high_school_us_history": 0.7093596059113301, "high_school_physics": 0.4, "high_school_world_history": 0.7542372881355932, "virology": 0.46060606060606063, "high_school_microeconomics": 0.569620253164557, "econometrics": 0.45132743362831856, "college_computer_science": 0.40404040404040403, "high_school_biology": 0.6731391585760518, "abstract_algebra": 0.2727272727272727, "professional_accounting": 0.38434163701067614, "philosophy": 0.6193548387096774, "professional_medicine": 0.5867158671586716, "nutrition": 0.6032786885245902, "global_facts": 0.32323232323232326, "machine_learning": 0.42342342342342343, "security_studies": 0.5614754098360656, "public_relations": 0.5871559633027523, "professional_psychology": 0.5171849427168577, "prehistory": 0.5727554179566563, "anatomy": 0.47761194029850745, "human_sexuality": 0.6615384615384615, "college_medicine": 0.5755813953488372, "high_school_government_and_politics": 0.7864583333333334, "college_chemistry": 0.40404040404040403, "logical_fallacies": 0.6172839506172839, "high_school_geography": 0.7411167512690355, "elementary_mathematics": 0.6259946949602122, "human_aging": 0.5765765765765766, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.7536764705882353, "formal_logic": 0.376, "high_school_statistics": 0.4232558139534884, "international_law": 0.6833333333333333, "high_school_mathematics": 0.26022304832713755, "high_school_computer_science": 0.6565656565656566, "conceptual_physics": 0.5299145299145299, "miscellaneous": 0.7519181585677749, "high_school_chemistry": 0.4207920792079208, "marketing": 0.8197424892703863, "professional_law": 0.4142204827136334, "management": 0.7058823529411765, "college_physics": 0.38613861386138615, "jurisprudence": 0.6915887850467289, "world_religions": 0.7529411764705882, "sociology": 0.75, "us_foreign_policy": 0.8585858585858586, "high_school_macroeconomics": 0.5424164524421594, "computer_security": 0.6666666666666666, "moral_scenarios": 0.40939597315436244, "moral_disputes": 0.5768115942028985, "electrical_engineering": 0.5138888888888888, "astronomy": 0.6291390728476821, "college_biology": 0.6083916083916084 } }, "prompt_2": { "accuracy": 0.5370039327851269, "category_acc": { "high_school_european_history": 0.75, "business_ethics": 0.6464646464646465, "clinical_knowledge": 0.5757575757575758, "medical_genetics": 0.6565656565656566, "high_school_us_history": 0.7044334975369458, "high_school_physics": 0.34, "high_school_world_history": 0.7330508474576272, "virology": 0.3939393939393939, "high_school_microeconomics": 0.5864978902953587, "econometrics": 0.415929203539823, "college_computer_science": 0.46464646464646464, "high_school_biology": 0.6699029126213593, "abstract_algebra": 0.32323232323232326, "professional_accounting": 0.3701067615658363, "philosophy": 0.5806451612903226, "professional_medicine": 0.5424354243542435, "nutrition": 0.5344262295081967, "global_facts": 0.3838383838383838, "machine_learning": 0.3963963963963964, "security_studies": 0.5860655737704918, "public_relations": 0.5779816513761468, "professional_psychology": 0.486088379705401, "prehistory": 0.5479876160990712, "anatomy": 0.4552238805970149, "human_sexuality": 0.6615384615384615, "college_medicine": 0.5406976744186046, "high_school_government_and_politics": 0.6510416666666666, "college_chemistry": 0.35353535353535354, "logical_fallacies": 0.6172839506172839, "high_school_geography": 0.6649746192893401, "elementary_mathematics": 0.6180371352785146, "human_aging": 0.536036036036036, "college_mathematics": 0.29292929292929293, "high_school_psychology": 0.7242647058823529, "formal_logic": 0.368, "high_school_statistics": 0.413953488372093, "international_law": 0.625, "high_school_mathematics": 0.241635687732342, "high_school_computer_science": 0.6262626262626263, "conceptual_physics": 0.5042735042735043, "miscellaneous": 0.7416879795396419, "high_school_chemistry": 0.4405940594059406, "marketing": 0.7896995708154506, "professional_law": 0.4005218525766471, "management": 0.696078431372549, "college_physics": 0.33663366336633666, "jurisprudence": 0.6822429906542056, "world_religions": 0.7058823529411765, "sociology": 0.65, "us_foreign_policy": 0.6868686868686869, "high_school_macroeconomics": 0.5115681233933161, "computer_security": 0.6666666666666666, "moral_scenarios": 0.3870246085011186, "moral_disputes": 0.48985507246376814, "electrical_engineering": 0.5277777777777778, "astronomy": 0.5629139072847682, "college_biology": 0.5734265734265734 } }, "prompt_3": { "accuracy": 0.5659635323560959, "category_acc": { "high_school_european_history": 0.7134146341463414, "business_ethics": 0.6464646464646465, "clinical_knowledge": 0.6136363636363636, "medical_genetics": 0.696969696969697, "high_school_us_history": 0.7241379310344828, "high_school_physics": 0.3466666666666667, "high_school_world_history": 0.7627118644067796, "virology": 0.4303030303030303, "high_school_microeconomics": 0.5822784810126582, "econometrics": 0.40707964601769914, "college_computer_science": 0.3838383838383838, "high_school_biology": 0.6699029126213593, "abstract_algebra": 0.3434343434343434, "professional_accounting": 0.40569395017793597, "philosophy": 0.6096774193548387, "professional_medicine": 0.6088560885608856, "nutrition": 0.6098360655737705, "global_facts": 0.30303030303030304, "machine_learning": 0.45045045045045046, "security_studies": 0.5737704918032787, "public_relations": 0.5688073394495413, "professional_psychology": 0.5384615384615384, "prehistory": 0.5758513931888545, "anatomy": 0.4925373134328358, "human_sexuality": 0.676923076923077, "college_medicine": 0.5988372093023255, "high_school_government_and_politics": 0.75, "college_chemistry": 0.45454545454545453, "logical_fallacies": 0.6604938271604939, "high_school_geography": 0.7461928934010152, "elementary_mathematics": 0.596816976127321, "human_aging": 0.6081081081081081, "college_mathematics": 0.2828282828282828, "high_school_psychology": 0.7702205882352942, "formal_logic": 0.432, "high_school_statistics": 0.4511627906976744, "international_law": 0.6583333333333333, "high_school_mathematics": 0.25650557620817843, "high_school_computer_science": 0.6161616161616161, "conceptual_physics": 0.5170940170940171, "miscellaneous": 0.7634271099744245, "high_school_chemistry": 0.47029702970297027, "marketing": 0.7854077253218884, "professional_law": 0.4227005870841487, "management": 0.7156862745098039, "college_physics": 0.3564356435643564, "jurisprudence": 0.6822429906542056, "world_religions": 0.7588235294117647, "sociology": 0.71, "us_foreign_policy": 0.8282828282828283, "high_school_macroeconomics": 0.5398457583547558, "computer_security": 0.6464646464646465, "moral_scenarios": 0.40380313199105144, "moral_disputes": 0.5884057971014492, "electrical_engineering": 0.4652777777777778, "astronomy": 0.6357615894039735, "college_biology": 0.6433566433566433 } }, "prompt_4": { "accuracy": 0.5445119771183411, "category_acc": { "high_school_european_history": 0.7378048780487805, "business_ethics": 0.6060606060606061, "clinical_knowledge": 0.5681818181818182, "medical_genetics": 0.6464646464646465, "high_school_us_history": 0.6600985221674877, "high_school_physics": 0.28, "high_school_world_history": 0.6949152542372882, "virology": 0.44242424242424244, "high_school_microeconomics": 0.5443037974683544, "econometrics": 0.36283185840707965, "college_computer_science": 0.35353535353535354, "high_school_biology": 0.6343042071197411, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.398576512455516, "philosophy": 0.5967741935483871, "professional_medicine": 0.5682656826568265, "nutrition": 0.5639344262295082, "global_facts": 0.40404040404040403, "machine_learning": 0.42342342342342343, "security_studies": 0.5122950819672131, "public_relations": 0.5688073394495413, "professional_psychology": 0.5286415711947627, "prehistory": 0.5758513931888545, "anatomy": 0.4253731343283582, "human_sexuality": 0.6692307692307692, "college_medicine": 0.5872093023255814, "high_school_government_and_politics": 0.734375, "college_chemistry": 0.45454545454545453, "logical_fallacies": 0.5987654320987654, "high_school_geography": 0.7258883248730964, "elementary_mathematics": 0.6153846153846154, "human_aging": 0.509009009009009, "college_mathematics": 0.25252525252525254, "high_school_psychology": 0.7169117647058824, "formal_logic": 0.448, "high_school_statistics": 0.37209302325581395, "international_law": 0.7083333333333334, "high_school_mathematics": 0.24907063197026022, "high_school_computer_science": 0.5959595959595959, "conceptual_physics": 0.5341880341880342, "miscellaneous": 0.7506393861892583, "high_school_chemistry": 0.4306930693069307, "marketing": 0.7982832618025751, "professional_law": 0.4142204827136334, "management": 0.6764705882352942, "college_physics": 0.37623762376237624, "jurisprudence": 0.6261682242990654, "world_religions": 0.7588235294117647, "sociology": 0.7, "us_foreign_policy": 0.7676767676767676, "high_school_macroeconomics": 0.5167095115681234, "computer_security": 0.6666666666666666, "moral_scenarios": 0.38814317673378074, "moral_disputes": 0.6028985507246377, "electrical_engineering": 0.5069444444444444, "astronomy": 0.6026490066225165, "college_biology": 0.5174825174825175 } }, "prompt_5": { "accuracy": 0.54315337861995, "category_acc": { "high_school_european_history": 0.6829268292682927, "business_ethics": 0.6565656565656566, "clinical_knowledge": 0.5681818181818182, "medical_genetics": 0.6262626262626263, "high_school_us_history": 0.6600985221674877, "high_school_physics": 0.32, "high_school_world_history": 0.690677966101695, "virology": 0.41818181818181815, "high_school_microeconomics": 0.540084388185654, "econometrics": 0.415929203539823, "college_computer_science": 0.43434343434343436, "high_school_biology": 0.6343042071197411, "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.3807829181494662, "philosophy": 0.5967741935483871, "professional_medicine": 0.5202952029520295, "nutrition": 0.580327868852459, "global_facts": 0.3838383838383838, "machine_learning": 0.43243243243243246, "security_studies": 0.5409836065573771, "public_relations": 0.6238532110091743, "professional_psychology": 0.502454991816694, "prehistory": 0.5572755417956656, "anatomy": 0.5298507462686567, "human_sexuality": 0.6461538461538462, "college_medicine": 0.5348837209302325, "high_school_government_and_politics": 0.78125, "college_chemistry": 0.36363636363636365, "logical_fallacies": 0.5925925925925926, "high_school_geography": 0.7055837563451777, "elementary_mathematics": 0.5623342175066313, "human_aging": 0.581081081081081, "college_mathematics": 0.32323232323232326, "high_school_psychology": 0.7518382352941176, "formal_logic": 0.384, "high_school_statistics": 0.4046511627906977, "international_law": 0.6583333333333333, "high_school_mathematics": 0.24535315985130113, "high_school_computer_science": 0.6767676767676768, "conceptual_physics": 0.49572649572649574, "miscellaneous": 0.7480818414322251, "high_school_chemistry": 0.4158415841584158, "marketing": 0.8068669527896996, "professional_law": 0.42661448140900193, "management": 0.7254901960784313, "college_physics": 0.39603960396039606, "jurisprudence": 0.6448598130841121, "world_religions": 0.7529411764705882, "sociology": 0.715, "us_foreign_policy": 0.7474747474747475, "high_school_macroeconomics": 0.4910025706940874, "computer_security": 0.6262626262626263, "moral_scenarios": 0.354586129753915, "moral_disputes": 0.5797101449275363, "electrical_engineering": 0.4722222222222222, "astronomy": 0.6556291390728477, "college_biology": 0.5874125874125874 } } }, "c_eval": { "prompt_1": { "accuracy": 0.6389301634472511 }, "prompt_2": { "accuracy": 0.6404160475482912 }, "prompt_3": { "accuracy": 0.6441307578008916 }, "prompt_4": { "accuracy": 0.5861812778603269 }, "prompt_5": { "accuracy": 0.524517087667162 } }, "c_eval_full": { "prompt_1": { "accuracy": 0.6550435865504358, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.625, "computer_architecture": 0.5769230769230769, "college_programming": 0.6428571428571429, "college_physics": 0.25, "college_chemistry": 0.4482758620689655, "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.4523809523809524, "metrology_engineer": 0.7586206896551724, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.6666666666666666, "high_school_chemistry": 0.75, "high_school_biology": 0.7916666666666666, "middle_school_mathematics": 0.4583333333333333, "middle_school_biology": 0.9230769230769231, "middle_school_physics": 0.875, "middle_school_chemistry": 0.92, "veterinary_medicine": 0.6071428571428571, "college_economics": 0.4666666666666667, "business_administration": 0.631578947368421, "marxism": 0.875, "mao_zedong_thought": 0.8620689655172413, "education_science": 0.7352941176470589, "teacher_qualification": 0.8163265306122449, "high_school_politics": 0.9166666666666666, "high_school_geography": 0.8333333333333334, "middle_school_politics": 0.9230769230769231, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.8571428571428571, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.5185185185185185, "law": 0.6896551724137931, "chinese_language_and_literature": 0.6428571428571429, "art_studies": 0.6842105263157895, "professional_tour_guide": 0.8235294117647058, "legal_professional": 0.5, "high_school_chinese": 0.625, "high_school_history": 0.88, "middle_school_history": 0.9629629629629629, "civil_servant": 0.5192307692307693, "sports_science": 0.7083333333333334, "plant_protection": 0.6666666666666666, "basic_medicine": 0.875, "clinical_medicine": 0.6666666666666666, "urban_and_rural_planner": 0.5882352941176471, "accountant": 0.5370370370370371, "fire_engineer": 0.5555555555555556, "environmental_impact_assessment_engineer": 0.6666666666666666, "tax_accountant": 0.5740740740740741, "physician": 0.6666666666666666 } }, "prompt_2": { "accuracy": 0.6475716064757161, "category_acc": { "computer_network": 0.5416666666666666, "operating_system": 0.5833333333333334, "computer_architecture": 0.6538461538461539, "college_programming": 0.6428571428571429, "college_physics": 0.2916666666666667, "college_chemistry": 0.4482758620689655, "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.42857142857142855, "metrology_engineer": 0.7931034482758621, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.5833333333333334, "high_school_chemistry": 0.6666666666666666, "high_school_biology": 0.75, "middle_school_mathematics": 0.5833333333333334, "middle_school_biology": 0.9230769230769231, "middle_school_physics": 0.9166666666666666, "middle_school_chemistry": 0.92, "veterinary_medicine": 0.6785714285714286, "college_economics": 0.55, "business_administration": 0.5789473684210527, "marxism": 0.875, "mao_zedong_thought": 0.7931034482758621, "education_science": 0.7058823529411765, "teacher_qualification": 0.7551020408163265, "high_school_politics": 0.9166666666666666, "high_school_geography": 0.8333333333333334, "middle_school_politics": 1.0, "middle_school_geography": 0.8823529411764706, "modern_chinese_history": 0.8571428571428571, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.4444444444444444, "law": 0.7241379310344828, "chinese_language_and_literature": 0.5357142857142857, "art_studies": 0.6578947368421053, "professional_tour_guide": 0.8235294117647058, "legal_professional": 0.42857142857142855, "high_school_chinese": 0.5833333333333334, "high_school_history": 0.84, "middle_school_history": 0.8888888888888888, "civil_servant": 0.5384615384615384, "sports_science": 0.7083333333333334, "plant_protection": 0.7037037037037037, "basic_medicine": 0.875, "clinical_medicine": 0.5925925925925926, "urban_and_rural_planner": 0.6078431372549019, "accountant": 0.5925925925925926, "fire_engineer": 0.5277777777777778, "environmental_impact_assessment_engineer": 0.6111111111111112, "tax_accountant": 0.5740740740740741, "physician": 0.6666666666666666 } }, "prompt_3": { "accuracy": 0.6687422166874222, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.5, "computer_architecture": 0.6538461538461539, "college_programming": 0.7142857142857143, "college_physics": 0.3333333333333333, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.13043478260869565, "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.35714285714285715, "metrology_engineer": 0.7586206896551724, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.6666666666666666, "high_school_chemistry": 0.7916666666666666, "high_school_biology": 0.7916666666666666, "middle_school_mathematics": 0.5833333333333334, "middle_school_biology": 0.9230769230769231, "middle_school_physics": 0.9166666666666666, "middle_school_chemistry": 0.88, "veterinary_medicine": 0.6428571428571429, "college_economics": 0.5666666666666667, "business_administration": 0.6842105263157895, "marxism": 0.8333333333333334, "mao_zedong_thought": 0.8275862068965517, "education_science": 0.7058823529411765, "teacher_qualification": 0.8367346938775511, "high_school_politics": 0.9166666666666666, "high_school_geography": 0.875, "middle_school_politics": 0.9615384615384616, "middle_school_geography": 0.9411764705882353, "modern_chinese_history": 0.8214285714285714, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.5185185185185185, "law": 0.6551724137931034, "chinese_language_and_literature": 0.6428571428571429, "art_studies": 0.6578947368421053, "professional_tour_guide": 0.7941176470588235, "legal_professional": 0.6071428571428571, "high_school_chinese": 0.7083333333333334, "high_school_history": 0.84, "middle_school_history": 0.9629629629629629, "civil_servant": 0.5384615384615384, "sports_science": 0.6666666666666666, "plant_protection": 0.7407407407407407, "basic_medicine": 0.8333333333333334, "clinical_medicine": 0.6666666666666666, "urban_and_rural_planner": 0.6274509803921569, "accountant": 0.6666666666666666, "fire_engineer": 0.6944444444444444, "environmental_impact_assessment_engineer": 0.6666666666666666, "tax_accountant": 0.6111111111111112, "physician": 0.6851851851851852 } }, "prompt_4": { "accuracy": 0.5772104607721046, "category_acc": { "computer_network": 0.4166666666666667, "operating_system": 0.4166666666666667, "computer_architecture": 0.5, "college_programming": 0.4523809523809524, "college_physics": 0.20833333333333334, "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.25, "probability_and_statistics": 0.17391304347826086, "discrete_mathematics": 0.3333333333333333, "electrical_engineer": 0.4523809523809524, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.625, "high_school_chemistry": 0.4166666666666667, "high_school_biology": 0.5833333333333334, "middle_school_mathematics": 0.375, "middle_school_biology": 0.8846153846153846, "middle_school_physics": 0.6666666666666666, "middle_school_chemistry": 0.76, "veterinary_medicine": 0.6071428571428571, "college_economics": 0.5666666666666667, "business_administration": 0.5789473684210527, "marxism": 0.7916666666666666, "mao_zedong_thought": 0.8275862068965517, "education_science": 0.6764705882352942, "teacher_qualification": 0.7959183673469388, "high_school_politics": 0.9166666666666666, "high_school_geography": 0.75, "middle_school_politics": 0.9615384615384616, "middle_school_geography": 0.8823529411764706, "modern_chinese_history": 0.75, "ideological_and_moral_cultivation": 0.9583333333333334, "logic": 0.25925925925925924, "law": 0.5172413793103449, "chinese_language_and_literature": 0.5357142857142857, "art_studies": 0.631578947368421, "professional_tour_guide": 0.8235294117647058, "legal_professional": 0.35714285714285715, "high_school_chinese": 0.375, "high_school_history": 0.8, "middle_school_history": 0.8148148148148148, "civil_servant": 0.4423076923076923, "sports_science": 0.4166666666666667, "plant_protection": 0.7037037037037037, "basic_medicine": 0.7916666666666666, "clinical_medicine": 0.7407407407407407, "urban_and_rural_planner": 0.5686274509803921, "accountant": 0.5555555555555556, "fire_engineer": 0.5277777777777778, "environmental_impact_assessment_engineer": 0.5555555555555556, "tax_accountant": 0.42592592592592593, "physician": 0.5740740740740741 } }, "prompt_5": { "accuracy": 0.5342465753424658, "category_acc": { "computer_network": 0.4583333333333333, "operating_system": 0.5416666666666666, "computer_architecture": 0.46153846153846156, "college_programming": 0.5238095238095238, "college_physics": 0.375, "college_chemistry": 0.3448275862068966, "advanced_mathematics": 0.4166666666666667, "probability_and_statistics": 0.2608695652173913, "discrete_mathematics": 0.23809523809523808, "electrical_engineer": 0.40476190476190477, "metrology_engineer": 0.5517241379310345, "high_school_mathematics": 0.43478260869565216, "high_school_physics": 0.5416666666666666, "high_school_chemistry": 0.5416666666666666, "high_school_biology": 0.6666666666666666, "middle_school_mathematics": 0.25, "middle_school_biology": 0.7692307692307693, "middle_school_physics": 0.5, "middle_school_chemistry": 0.72, "veterinary_medicine": 0.5, "college_economics": 0.4666666666666667, "business_administration": 0.47368421052631576, "marxism": 0.75, "mao_zedong_thought": 0.6206896551724138, "education_science": 0.6470588235294118, "teacher_qualification": 0.6530612244897959, "high_school_politics": 0.5416666666666666, "high_school_geography": 0.6666666666666666, "middle_school_politics": 0.7307692307692307, "middle_school_geography": 0.8823529411764706, "modern_chinese_history": 0.6428571428571429, "ideological_and_moral_cultivation": 0.7083333333333334, "logic": 0.3333333333333333, "law": 0.5517241379310345, "chinese_language_and_literature": 0.6428571428571429, "art_studies": 0.6052631578947368, "professional_tour_guide": 0.7352941176470589, "legal_professional": 0.32142857142857145, "high_school_chinese": 0.375, "high_school_history": 0.64, "middle_school_history": 0.8148148148148148, "civil_servant": 0.4423076923076923, "sports_science": 0.375, "plant_protection": 0.5925925925925926, "basic_medicine": 0.6666666666666666, "clinical_medicine": 0.4074074074074074, "urban_and_rural_planner": 0.5882352941176471, "accountant": 0.3888888888888889, "fire_engineer": 0.5, "environmental_impact_assessment_engineer": 0.6666666666666666, "tax_accountant": 0.35185185185185186, "physician": 0.6851851851851852 } } }, "cmmlu": { "prompt_1": { "accuracy": 0.6845878136200717 }, "prompt_2": { "accuracy": 0.6810035842293907 }, "prompt_3": { "accuracy": 0.6774193548387096 }, "prompt_4": { "accuracy": 0.6702508960573477 }, "prompt_5": { "accuracy": 0.5949820788530465 } }, "cmmlu_full": { "prompt_1": { "accuracy": 0.6518735969608013, "category_acc": { "agronomy": 0.514792899408284, "anatomy": 0.6418918918918919, "ancient_chinese": 0.4024390243902439, "arts": 0.85, "astronomy": 0.4121212121212121, "business_ethics": 0.631578947368421, "chinese_civil_service_exam": 0.5875, "chinese_driving_rule": 0.8854961832061069, "chinese_food_culture": 0.6323529411764706, "chinese_foreign_policy": 0.7009345794392523, "chinese_history": 0.7832817337461301, "chinese_literature": 0.4950980392156863, "chinese_teacher_qualification": 0.8212290502793296, "clinical_knowledge": 0.6118143459915611, "college_actuarial_science": 0.2830188679245283, "college_education": 0.794392523364486, "college_engineering_hydrology": 0.6226415094339622, "college_law": 0.6296296296296297, "college_mathematics": 0.3047619047619048, "college_medical_statistics": 0.5188679245283019, "college_medicine": 0.673992673992674, "computer_science": 0.6519607843137255, "computer_security": 0.7660818713450293, "conceptual_physics": 0.8163265306122449, "construction_project_management": 0.5539568345323741, "economics": 0.6352201257861635, "education": 0.6871165644171779, "electrical_engineering": 0.6337209302325582, "elementary_chinese": 0.6666666666666666, "elementary_commonsense": 0.6767676767676768, "elementary_information_and_technology": 0.8235294117647058, "elementary_mathematics": 0.48695652173913045, "ethnology": 0.6666666666666666, "food_science": 0.5734265734265734, "genetics": 0.5625, "global_facts": 0.6442953020134228, "high_school_biology": 0.757396449704142, "high_school_chemistry": 0.6893939393939394, "high_school_geography": 0.788135593220339, "high_school_mathematics": 0.35365853658536583, "high_school_physics": 0.6909090909090909, "high_school_politics": 0.7132867132867133, "human_sexuality": 0.6190476190476191, "international_law": 0.5405405405405406, "journalism": 0.6046511627906976, "jurisprudence": 0.6788321167883211, "legal_and_moral_basis": 0.9485981308411215, "logical": 0.5203252032520326, "machine_learning": 0.5409836065573771, "management": 0.7714285714285715, "marketing": 0.7055555555555556, "marxist_theory": 0.8201058201058201, "modern_chinese": 0.5344827586206896, "nutrition": 0.6275862068965518, "philosophy": 0.6476190476190476, "professional_accounting": 0.7314285714285714, "professional_law": 0.5355450236966824, "professional_medicine": 0.5292553191489362, "professional_psychology": 0.7801724137931034, "public_relations": 0.6379310344827587, "security_study": 0.7333333333333333, "sociology": 0.6637168141592921, "sports_science": 0.6606060606060606, "traditional_chinese_medicine": 0.6702702702702703, "virology": 0.650887573964497, "world_history": 0.7763975155279503, "world_religions": 0.70625 } }, "prompt_2": { "accuracy": 0.6434121913313763, "category_acc": { "agronomy": 0.5384615384615384, "anatomy": 0.5945945945945946, "ancient_chinese": 0.3719512195121951, "arts": 0.85625, "astronomy": 0.4303030303030303, "business_ethics": 0.6220095693779905, "chinese_civil_service_exam": 0.60625, "chinese_driving_rule": 0.8854961832061069, "chinese_food_culture": 0.6176470588235294, "chinese_foreign_policy": 0.7102803738317757, "chinese_history": 0.7708978328173375, "chinese_literature": 0.4950980392156863, "chinese_teacher_qualification": 0.8100558659217877, "clinical_knowledge": 0.5949367088607594, "college_actuarial_science": 0.32075471698113206, "college_education": 0.7757009345794392, "college_engineering_hydrology": 0.5188679245283019, "college_law": 0.5925925925925926, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.4811320754716981, "college_medicine": 0.6410256410256411, "computer_science": 0.6813725490196079, "computer_security": 0.7602339181286549, "conceptual_physics": 0.8027210884353742, "construction_project_management": 0.5539568345323741, "economics": 0.6477987421383647, "education": 0.6748466257668712, "electrical_engineering": 0.6453488372093024, "elementary_chinese": 0.6626984126984127, "elementary_commonsense": 0.6818181818181818, "elementary_information_and_technology": 0.8319327731092437, "elementary_mathematics": 0.4956521739130435, "ethnology": 0.6666666666666666, "food_science": 0.6153846153846154, "genetics": 0.5340909090909091, "global_facts": 0.6375838926174496, "high_school_biology": 0.7041420118343196, "high_school_chemistry": 0.6666666666666666, "high_school_geography": 0.7627118644067796, "high_school_mathematics": 0.38414634146341464, "high_school_physics": 0.6181818181818182, "high_school_politics": 0.6923076923076923, "human_sexuality": 0.5952380952380952, "international_law": 0.5567567567567567, "journalism": 0.6046511627906976, "jurisprudence": 0.6788321167883211, "legal_and_moral_basis": 0.9392523364485982, "logical": 0.5447154471544715, "machine_learning": 0.4918032786885246, "management": 0.7761904761904762, "marketing": 0.6888888888888889, "marxist_theory": 0.8148148148148148, "modern_chinese": 0.5603448275862069, "nutrition": 0.6482758620689655, "philosophy": 0.638095238095238, "professional_accounting": 0.7028571428571428, "professional_law": 0.5639810426540285, "professional_medicine": 0.5132978723404256, "professional_psychology": 0.771551724137931, "public_relations": 0.6494252873563219, "security_study": 0.6888888888888889, "sociology": 0.6371681415929203, "sports_science": 0.6484848484848484, "traditional_chinese_medicine": 0.6432432432432432, "virology": 0.6272189349112426, "world_history": 0.7080745341614907, "world_religions": 0.69375 } }, "prompt_3": { "accuracy": 0.6450526679329995, "category_acc": { "agronomy": 0.5207100591715976, "anatomy": 0.6148648648648649, "ancient_chinese": 0.40853658536585363, "arts": 0.85625, "astronomy": 0.38181818181818183, "business_ethics": 0.6124401913875598, "chinese_civil_service_exam": 0.6, "chinese_driving_rule": 0.8854961832061069, "chinese_food_culture": 0.6029411764705882, "chinese_foreign_policy": 0.7102803738317757, "chinese_history": 0.7832817337461301, "chinese_literature": 0.49019607843137253, "chinese_teacher_qualification": 0.8044692737430168, "clinical_knowledge": 0.620253164556962, "college_actuarial_science": 0.3867924528301887, "college_education": 0.7663551401869159, "college_engineering_hydrology": 0.5188679245283019, "college_law": 0.6018518518518519, "college_mathematics": 0.2857142857142857, "college_medical_statistics": 0.5377358490566038, "college_medicine": 0.663003663003663, "computer_science": 0.6666666666666666, "computer_security": 0.7602339181286549, "conceptual_physics": 0.8163265306122449, "construction_project_management": 0.5755395683453237, "economics": 0.6792452830188679, "education": 0.6503067484662577, "electrical_engineering": 0.6511627906976745, "elementary_chinese": 0.6507936507936508, "elementary_commonsense": 0.6717171717171717, "elementary_information_and_technology": 0.8151260504201681, "elementary_mathematics": 0.49130434782608695, "ethnology": 0.6370370370370371, "food_science": 0.5804195804195804, "genetics": 0.5170454545454546, "global_facts": 0.6644295302013423, "high_school_biology": 0.7218934911242604, "high_school_chemistry": 0.6742424242424242, "high_school_geography": 0.7627118644067796, "high_school_mathematics": 0.36585365853658536, "high_school_physics": 0.5727272727272728, "high_school_politics": 0.7412587412587412, "human_sexuality": 0.6111111111111112, "international_law": 0.5135135135135135, "journalism": 0.5988372093023255, "jurisprudence": 0.6666666666666666, "legal_and_moral_basis": 0.9439252336448598, "logical": 0.5528455284552846, "machine_learning": 0.4672131147540984, "management": 0.7619047619047619, "marketing": 0.7166666666666667, "marxist_theory": 0.7936507936507936, "modern_chinese": 0.5172413793103449, "nutrition": 0.6413793103448275, "philosophy": 0.6666666666666666, "professional_accounting": 0.7257142857142858, "professional_law": 0.5639810426540285, "professional_medicine": 0.5292553191489362, "professional_psychology": 0.7758620689655172, "public_relations": 0.6666666666666666, "security_study": 0.7333333333333333, "sociology": 0.6238938053097345, "sports_science": 0.6545454545454545, "traditional_chinese_medicine": 0.654054054054054, "virology": 0.6449704142011834, "world_history": 0.7453416149068323, "world_religions": 0.6875 } }, "prompt_4": { "accuracy": 0.60145052667933, "category_acc": { "agronomy": 0.5266272189349113, "anatomy": 0.6013513513513513, "ancient_chinese": 0.3353658536585366, "arts": 0.85, "astronomy": 0.4121212121212121, "business_ethics": 0.5933014354066986, "chinese_civil_service_exam": 0.5125, "chinese_driving_rule": 0.8015267175572519, "chinese_food_culture": 0.6764705882352942, "chinese_foreign_policy": 0.6074766355140186, "chinese_history": 0.6687306501547987, "chinese_literature": 0.5147058823529411, "chinese_teacher_qualification": 0.7988826815642458, "clinical_knowledge": 0.5358649789029536, "college_actuarial_science": 0.25471698113207547, "college_education": 0.7383177570093458, "college_engineering_hydrology": 0.5377358490566038, "college_law": 0.5555555555555556, "college_mathematics": 0.3142857142857143, "college_medical_statistics": 0.49056603773584906, "college_medicine": 0.6556776556776557, "computer_science": 0.6372549019607843, "computer_security": 0.7017543859649122, "conceptual_physics": 0.673469387755102, "construction_project_management": 0.5251798561151079, "economics": 0.5911949685534591, "education": 0.6380368098159509, "electrical_engineering": 0.5988372093023255, "elementary_chinese": 0.6706349206349206, "elementary_commonsense": 0.6515151515151515, "elementary_information_and_technology": 0.8109243697478992, "elementary_mathematics": 0.4043478260869565, "ethnology": 0.674074074074074, "food_science": 0.5594405594405595, "genetics": 0.45454545454545453, "global_facts": 0.610738255033557, "high_school_biology": 0.5562130177514792, "high_school_chemistry": 0.44696969696969696, "high_school_geography": 0.6949152542372882, "high_school_mathematics": 0.3780487804878049, "high_school_physics": 0.4636363636363636, "high_school_politics": 0.6433566433566433, "human_sexuality": 0.5317460317460317, "international_law": 0.5243243243243243, "journalism": 0.563953488372093, "jurisprudence": 0.5985401459854015, "legal_and_moral_basis": 0.9205607476635514, "logical": 0.5121951219512195, "machine_learning": 0.5081967213114754, "management": 0.6952380952380952, "marketing": 0.65, "marxist_theory": 0.7619047619047619, "modern_chinese": 0.5086206896551724, "nutrition": 0.6344827586206897, "philosophy": 0.6, "professional_accounting": 0.68, "professional_law": 0.46919431279620855, "professional_medicine": 0.5212765957446809, "professional_psychology": 0.7155172413793104, "public_relations": 0.6379310344827587, "security_study": 0.674074074074074, "sociology": 0.6061946902654868, "sports_science": 0.593939393939394, "traditional_chinese_medicine": 0.6162162162162163, "virology": 0.6094674556213018, "world_history": 0.6335403726708074, "world_religions": 0.675 } }, "prompt_5": { "accuracy": 0.5436021412536695, "category_acc": { "agronomy": 0.46153846153846156, "anatomy": 0.581081081081081, "ancient_chinese": 0.32926829268292684, "arts": 0.775, "astronomy": 0.4303030303030303, "business_ethics": 0.5119617224880383, "chinese_civil_service_exam": 0.41875, "chinese_driving_rule": 0.7404580152671756, "chinese_food_culture": 0.6617647058823529, "chinese_foreign_policy": 0.45794392523364486, "chinese_history": 0.5975232198142415, "chinese_literature": 0.46568627450980393, "chinese_teacher_qualification": 0.7541899441340782, "clinical_knowledge": 0.4641350210970464, "college_actuarial_science": 0.32075471698113206, "college_education": 0.6355140186915887, "college_engineering_hydrology": 0.5, "college_law": 0.5462962962962963, "college_mathematics": 0.24761904761904763, "college_medical_statistics": 0.4528301886792453, "college_medicine": 0.4945054945054945, "computer_science": 0.5784313725490197, "computer_security": 0.5964912280701754, "conceptual_physics": 0.564625850340136, "construction_project_management": 0.5251798561151079, "economics": 0.5534591194968553, "education": 0.5705521472392638, "electrical_engineering": 0.5116279069767442, "elementary_chinese": 0.6031746031746031, "elementary_commonsense": 0.5959595959595959, "elementary_information_and_technology": 0.726890756302521, "elementary_mathematics": 0.4608695652173913, "ethnology": 0.5481481481481482, "food_science": 0.5804195804195804, "genetics": 0.4602272727272727, "global_facts": 0.5973154362416108, "high_school_biology": 0.5384615384615384, "high_school_chemistry": 0.45454545454545453, "high_school_geography": 0.5932203389830508, "high_school_mathematics": 0.2865853658536585, "high_school_physics": 0.45454545454545453, "high_school_politics": 0.5734265734265734, "human_sexuality": 0.5158730158730159, "international_law": 0.4810810810810811, "journalism": 0.5697674418604651, "jurisprudence": 0.5401459854014599, "legal_and_moral_basis": 0.794392523364486, "logical": 0.45528455284552843, "machine_learning": 0.47540983606557374, "management": 0.5952380952380952, "marketing": 0.6166666666666667, "marxist_theory": 0.6878306878306878, "modern_chinese": 0.4827586206896552, "nutrition": 0.5517241379310345, "philosophy": 0.5714285714285714, "professional_accounting": 0.5885714285714285, "professional_law": 0.3933649289099526, "professional_medicine": 0.42021276595744683, "professional_psychology": 0.6724137931034483, "public_relations": 0.5172413793103449, "security_study": 0.6592592592592592, "sociology": 0.5575221238938053, "sports_science": 0.5333333333333333, "traditional_chinese_medicine": 0.4972972972972973, "virology": 0.5384615384615384, "world_history": 0.6024844720496895, "world_religions": 0.64375 } } }, "zbench": { "prompt_1": { "accuracy": 0.42424242424242425 }, "prompt_2": { "accuracy": 0.5454545454545454 }, "prompt_3": { "accuracy": 0.48484848484848486 }, "prompt_4": { "accuracy": 0.36363636363636365 }, "prompt_5": { "accuracy": 0.5151515151515151 } }, "ind_emotion": { "prompt_1": { "accuracy": 0.5931818181818181 }, "prompt_2": { "accuracy": 0.47954545454545455 }, "prompt_3": { "accuracy": 0.5113636363636364 }, "prompt_4": { "accuracy": 0.5272727272727272 }, "prompt_5": { "accuracy": 0.5318181818181819 } }, "ocnli": { "prompt_1": { "accuracy": 0.6101694915254238 }, "prompt_2": { "accuracy": 0.5976271186440678 }, "prompt_3": { "accuracy": 0.5922033898305085 }, "prompt_4": { "accuracy": 0.5728813559322034 }, "prompt_5": { "accuracy": 0.5820338983050848 } }, "c3": { "prompt_1": { "accuracy": 0.8915482423335827 }, "prompt_2": { "accuracy": 0.8900523560209425 }, "prompt_3": { "accuracy": 0.8848167539267016 }, "prompt_4": { "accuracy": 0.8915482423335827 }, "prompt_5": { "accuracy": 0.8433059087509349 } }, "dream": { "prompt_1": { "accuracy": 0.8887800097991181 }, "prompt_2": { "accuracy": 0.8912297893189612 }, "prompt_3": { "accuracy": 0.8995590396864283 }, "prompt_4": { "accuracy": 0.8731014208721215 }, "prompt_5": { "accuracy": 0.8887800097991181 } }, "samsum": { "prompt_1": { "rouge1": 0.32025871790184207, "rouge2": 0.11252191613209159, "rougeL": 0.23687649729124394, "avg_rouge": 0.2232190437750592 }, "prompt_2": { "rouge1": 0.3539650693173222, "rouge2": 0.12289059669247782, "rougeL": 0.2621425399352911, "avg_rouge": 0.24633273531503033 }, "prompt_3": { "rouge1": 0.3596383366340126, "rouge2": 0.12306825565807013, "rougeL": 0.2667316013053415, "avg_rouge": 0.2498127311991414 }, "prompt_4": { "rouge1": 0.32746103807533683, "rouge2": 0.11032578135156446, "rougeL": 0.24118472863418863, "avg_rouge": 0.22632384935369665 }, "prompt_5": { "rouge1": 0.35920934397741366, "rouge2": 0.12008368532113817, "rougeL": 0.26394933485526906, "avg_rouge": 0.24774745471794027 } }, "dialogsum": { "prompt_1": { "rouge1": 0.2156449509326601, "rouge2": 0.051271372877158834, "rougeL": 0.15762523547298707, "avg_rouge": 0.14151385309426867 }, "prompt_2": { "rouge1": 0.21196166804340755, "rouge2": 0.048452163740065, "rougeL": 0.1531021308500753, "avg_rouge": 0.1378386542111826 }, "prompt_3": { "rouge1": 0.21407338332262357, "rouge2": 0.050122199903020484, "rougeL": 0.1556449616356449, "avg_rouge": 0.13994684828709633 }, "prompt_4": { "rouge1": 0.21417352959751168, "rouge2": 0.05186922938802024, "rougeL": 0.15696263445646286, "avg_rouge": 0.14100179781399827 }, "prompt_5": { "rouge1": 0.2197022831312668, "rouge2": 0.05280379885981328, "rougeL": 0.1617012052571708, "avg_rouge": 0.14473576241608363 } }, "sst2": { "prompt_1": { "accuracy": 0.9311926605504587 }, "prompt_2": { "accuracy": 0.9071100917431193 }, "prompt_3": { "accuracy": 0.9174311926605505 }, "prompt_4": { "accuracy": 0.9311926605504587 }, "prompt_5": { "accuracy": 0.8623853211009175 } }, "cola": { "prompt_1": { "accuracy": 0.822627037392138 }, "prompt_2": { "accuracy": 0.8149568552253116 }, "prompt_3": { "accuracy": 0.8207094918504314 }, "prompt_4": { "accuracy": 0.8159156279961649 }, "prompt_5": { "accuracy": 0.8139980824544583 } }, "qqp": { "prompt_1": { "accuracy": 0.7805 }, "prompt_2": { "accuracy": 0.7745 }, "prompt_3": { "accuracy": 0.7805 }, "prompt_4": { "accuracy": 0.7885 }, "prompt_5": { "accuracy": 0.7715 } }, "mnli": { "prompt_1": { "accuracy": 0.7 }, "prompt_2": { "accuracy": 0.7 }, "prompt_3": { "accuracy": 0.8 }, "prompt_4": { "accuracy": 0.7 }, "prompt_5": { "accuracy": 0.7 } }, "qnli": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.8 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.9 } }, "wnli": { "prompt_1": { "accuracy": 0.5 }, "prompt_2": { "accuracy": 0.5 }, "prompt_3": { "accuracy": 0.5 }, "prompt_4": { "accuracy": 0.4 }, "prompt_5": { "accuracy": 0.8 } }, "rte": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 1.0 }, "prompt_3": { "accuracy": 1.0 }, "prompt_4": { "accuracy": 0.8 }, "prompt_5": { "accuracy": 0.9 } }, "mrpc": { "prompt_1": { "accuracy": 0.9 }, "prompt_2": { "accuracy": 0.9 }, "prompt_3": { "accuracy": 0.9 }, "prompt_4": { "accuracy": 0.9 }, "prompt_5": { "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { "prompt_1": -1 }, "cross_mmlu": { "prompt_1": -1 }, "cross_logiqa": { "prompt_1": -1 }, "sg_eval": { "prompt_1": -1 }, "cn_eval": { "prompt_1": -1 }, "us_eval": { "prompt_1": -1 }, "ph_eval": { "prompt_1": -1 }, "sing2eng": { "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { "prompt_1": -1 }, "flores_vie2eng": { "prompt_1": -1 }, "flores_zho2eng": { "prompt_1": -1 }, "flores_zsm2eng": { "prompt_1": -1 }, "mmlu": { "prompt_1": -1 }, "mmlu_full": { "prompt_1": -1 }, "c_eval": { "prompt_1": -1 }, "c_eval_full": { "prompt_1": -1 }, "cmmlu": { "prompt_1": -1 }, "cmmlu_full": { "prompt_1": -1 }, "zbench": { "prompt_1": -1 }, "ind_emotion": { "prompt_1": -1 }, "ocnli": { "prompt_1": -1 }, "c3": { "prompt_1": -1 }, "dream": { "prompt_1": -1 }, "samsum": { "prompt_1": -1 }, "dialogsum": { "prompt_1": -1 }, "sst2": { "prompt_1": -1 }, "cola": { "prompt_1": -1 }, "qqp": { "prompt_1": -1 }, "mnli": { "prompt_1": -1 }, "qnli": { "prompt_1": -1 }, "wnli": { "prompt_1": -1 }, "rte": { "prompt_1": -1 }, "mrpc": { "prompt_1": -1 } } } }